From 1485f58a0710a228b65877c9f53f513dfc973e82 Mon Sep 17 00:00:00 2001 From: VinZch <220566909+VinZch@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:28:57 +0800 Subject: [PATCH] add reproducible sub-one-pass optimizer benchmark --- ...43\347\240\201\345\256\241\346\237\245.md" | 64 + ...71\346\241\210\350\256\276\350\256\241.md" | 117 + ...00\346\261\202\347\220\206\350\247\243.md" | 67 + .../formal/confirm_selection.json | 116 + .../data_manifests/fineweb-512m-manifest.json | 17 + .../data_manifests/preflight-manifest.json | 9 + .../formal/formal-confirm-pipeline.log | 1215 +++ exps/sub_one_pass/formal/formal-pipeline.log | 9040 +++++++++++++++++ .../sub_one_pass/formal/manifest_confirm.json | 766 ++ exps/sub_one_pass/formal/manifest_main.json | 1778 ++++ exps/sub_one_pass/formal/manifest_tune.json | 7540 ++++++++++++++ exps/sub_one_pass/formal/pipeline_state.json | 23 + .../formal/selected_lr_scales.json | 22 + .../formal/selected_lr_scales_report.json | 552 + .../run.log | 156 + .../summary.json | 1 + .../run.log | 156 + .../summary.json | 1 + .../run.log | 147 + .../summary.json | 1 + .../run.log | 147 + .../summary.json | 1 + .../run.log | 140 + .../summary.json | 1 + .../run.log | 140 + .../summary.json | 1 + .../run.log | 155 + .../summary.json | 1 + .../run.log | 155 + .../summary.json | 1 + .../run.log | 151 + .../summary.json | 1 + .../run.log | 156 + .../summary.json | 1 + .../run.log | 147 + .../summary.json | 1 + .../run.log | 143 + .../summary.json | 1 + .../run.log | 150 + .../summary.json | 1 + .../run.log | 141 + .../summary.json | 1 + .../run.log | 139 + .../summary.json | 1 + .../run.log | 151 + .../summary.json | 1 + .../run.log | 129 + .../summary.json | 1 + .../run.log | 140 + .../summary.json | 1 + .../run.log | 247 + .../summary.json | 1 + .../run.log | 146 + .../summary.json | 1 + .../run.log | 146 + .../summary.json | 1 + .../run.log | 159 + .../summary.json | 1 + .../run.log | 157 + .../summary.json | 1 + .../run.log | 151 + .../summary.json | 1 + .../run.log | 145 + .../summary.json | 1 + .../run.log | 145 + .../summary.json | 1 + .../run.log | 155 + .../summary.json | 1 + .../run.log | 145 + .../summary.json | 1 + .../run.log | 71 + .../summary.json | 1 + .../run.log | 71 + .../summary.json | 1 + .../run.log | 71 + .../summary.json | 1 + .../run.log | 71 + .../summary.json | 1 + .../run.log | 76 + .../summary.json | 1 + .../run.log | 76 + .../summary.json | 1 + .../run.log | 76 + .../summary.json | 1 + .../run.log | 67 + .../summary.json | 1 + .../run.log | 67 + .../summary.json | 1 + .../run.log | 67 + .../summary.json | 1 + .../run.log | 63 + .../summary.json | 1 + .../run.log | 63 + .../summary.json | 1 + .../run.log | 63 + .../summary.json | 1 + .../run.log | 63 + .../summary.json | 1 + .../run.log | 70 + .../summary.json | 1 + .../run.log | 70 + .../summary.json | 1 + .../run.log | 70 + .../summary.json | 1 + .../run.log | 70 + .../summary.json | 1 + .../run.log | 61 + .../summary.json | 1 + .../run.log | 61 + .../summary.json | 1 + .../run.log | 61 + .../summary.json | 1 + .../run.log | 61 + .../summary.json | 1 + .../run.log | 61 + .../summary.json | 1 + .../run.log | 61 + .../summary.json | 1 + .../run.log | 59 + .../summary.json | 1 + .../run.log | 59 + .../summary.json | 1 + .../run.log | 59 + .../summary.json | 1 + .../run.log | 59 + .../summary.json | 1 + .../run.log | 71 + .../summary.json | 1 + .../run.log | 71 + .../summary.json | 1 + .../run.log | 71 + .../summary.json | 1 + .../run.log | 71 + .../summary.json | 1 + .../run.log | 49 + .../summary.json | 1 + .../run.log | 49 + .../summary.json | 1 + .../run.log | 49 + .../summary.json | 1 + .../run.log | 49 + .../summary.json | 1 + .../run.log | 60 + .../summary.json | 1 + .../run.log | 60 + .../summary.json | 1 + .../run.log | 60 + .../summary.json | 1 + .../run.log | 60 + .../summary.json | 1 + .../run.log | 91 + .../summary.json | 1 + .../run.log | 91 + .../summary.json | 1 + .../run.log | 91 + .../summary.json | 1 + .../run.log | 91 + .../summary.json | 1 + .../run.log | 66 + .../summary.json | 1 + .../run.log | 66 + .../summary.json | 1 + .../run.log | 66 + .../summary.json | 1 + .../run.log | 66 + .../summary.json | 1 + .../run.log | 66 + .../summary.json | 1 + .../run.log | 66 + .../summary.json | 1 + .../run.log | 66 + .../summary.json | 1 + .../run.log | 66 + .../summary.json | 1 + .../run.log | 79 + .../summary.json | 1 + .../run.log | 79 + .../summary.json | 1 + .../run.log | 79 + .../summary.json | 1 + .../run.log | 77 + .../summary.json | 1 + .../run.log | 77 + .../summary.json | 1 + .../run.log | 77 + .../summary.json | 1 + .../run.log | 77 + .../summary.json | 1 + .../run.log | 77 + .../summary.json | 1 + .../run.log | 77 + .../summary.json | 1 + .../run.log | 77 + .../summary.json | 1 + .../run.log | 77 + .../summary.json | 1 + .../run.log | 71 + .../summary.json | 1 + .../run.log | 71 + .../summary.json | 1 + .../run.log | 71 + .../summary.json | 1 + .../run.log | 71 + .../summary.json | 1 + .../run.log | 71 + .../summary.json | 1 + .../run.log | 65 + .../summary.json | 1 + .../run.log | 65 + .../summary.json | 1 + .../run.log | 65 + .../summary.json | 1 + .../run.log | 65 + .../summary.json | 1 + .../run.log | 65 + .../summary.json | 1 + .../run.log | 65 + .../summary.json | 1 + .../run.log | 65 + .../summary.json | 1 + .../run.log | 75 + .../summary.json | 1 + .../run.log | 75 + .../summary.json | 1 + .../run.log | 75 + .../summary.json | 1 + .../run.log | 65 + .../summary.json | 1 + .../run.log | 65 + .../summary.json | 1 + .../run.log | 65 + .../summary.json | 1 + .../run.log | 65 + .../summary.json | 1 + .../formal/sub_one_pass_results.csv | 223 + scripts/sub_one_pass/README.md | 139 + scripts/sub_one_pass/estimate_runtime.py | 58 + .../sub_one_pass/prepare_fineweb_parquet.py | 102 + .../sub_one_pass/prepare_fineweb_stream.py | 126 + .../sub_one_pass/prepare_preflight_data.py | 64 + scripts/sub_one_pass/run_all.py | 598 ++ .../sub_one_pass/run_confirm_after_main.py | 230 + scripts/sub_one_pass/run_main_stage.py | 90 + scripts/sub_one_pass/run_server_pipeline.py | 200 + scripts/sub_one_pass/select_lr.py | 100 + scripts/sub_one_pass/summarize.py | 77 + src/config/base.py | 44 +- src/data/reader.py | 174 + src/data/token_bin.py | 12 + src/data/utils.py | 141 +- src/experiment_budget.py | 89 + src/main.py | 119 +- src/optim/base.py | 69 +- src/optim/composite.py | 79 + src/optim/schedule.py | 18 + tests/test_composite_optimizer.py | 55 + tests/test_data_reader.py | 42 + tests/test_experiment_budget.py | 50 + tests/test_fineweb_stream.py | 50 + tests/test_schedule.py | 24 + tests/test_sub_one_pass_runner.py | 112 + 261 files changed, 34219 insertions(+), 148 deletions(-) create mode 100644 "docs/sub_one_pass/\344\273\243\347\240\201\345\256\241\346\237\245.md" create mode 100644 "docs/sub_one_pass/\346\226\271\346\241\210\350\256\276\350\256\241.md" create mode 100644 "docs/sub_one_pass/\351\234\200\346\261\202\347\220\206\350\247\243.md" create mode 100644 exps/sub_one_pass/formal/confirm_selection.json create mode 100644 exps/sub_one_pass/formal/data_manifests/fineweb-512m-manifest.json create mode 100644 exps/sub_one_pass/formal/data_manifests/preflight-manifest.json create mode 100644 exps/sub_one_pass/formal/formal-confirm-pipeline.log create mode 100644 exps/sub_one_pass/formal/formal-pipeline.log create mode 100644 exps/sub_one_pass/formal/manifest_confirm.json create mode 100644 exps/sub_one_pass/formal/manifest_main.json create mode 100644 exps/sub_one_pass/formal/manifest_tune.json create mode 100644 exps/sub_one_pass/formal/pipeline_state.json create mode 100644 exps/sub_one_pass/formal/selected_lr_scales.json create mode 100644 exps/sub_one_pass/formal/selected_lr_scales_report.json create mode 100644 exps/sub_one_pass/formal/sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456/run.log create mode 100644 exps/sub_one_pass/formal/sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216/run.log create mode 100644 exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216/summary.json create mode 100644 exps/sub_one_pass/formal/sub_one_pass_results.csv create mode 100644 scripts/sub_one_pass/README.md create mode 100644 scripts/sub_one_pass/estimate_runtime.py create mode 100644 scripts/sub_one_pass/prepare_fineweb_parquet.py create mode 100644 scripts/sub_one_pass/prepare_fineweb_stream.py create mode 100644 scripts/sub_one_pass/prepare_preflight_data.py create mode 100644 scripts/sub_one_pass/run_all.py create mode 100644 scripts/sub_one_pass/run_confirm_after_main.py create mode 100644 scripts/sub_one_pass/run_main_stage.py create mode 100644 scripts/sub_one_pass/run_server_pipeline.py create mode 100644 scripts/sub_one_pass/select_lr.py create mode 100644 scripts/sub_one_pass/summarize.py create mode 100644 src/data/reader.py create mode 100644 src/data/token_bin.py create mode 100644 src/experiment_budget.py create mode 100644 src/optim/composite.py create mode 100644 tests/test_composite_optimizer.py create mode 100644 tests/test_data_reader.py create mode 100644 tests/test_experiment_budget.py create mode 100644 tests/test_fineweb_stream.py create mode 100644 tests/test_schedule.py create mode 100644 tests/test_sub_one_pass_runner.py diff --git "a/docs/sub_one_pass/\344\273\243\347\240\201\345\256\241\346\237\245.md" "b/docs/sub_one_pass/\344\273\243\347\240\201\345\256\241\346\237\245.md" new file mode 100644 index 0000000..0298ff6 --- /dev/null +++ "b/docs/sub_one_pass/\344\273\243\347\240\201\345\256\241\346\237\245.md" @@ -0,0 +1,64 @@ +## 2026-07-28 14:37 + +## 代码审查报告 + +### 审查目标 + +- 类型：本地改动 +- 范围：sub-one-pass token budget、数据读取、20-optimizer runner、结果汇总与实验文档 +- 审查时间：2026-07-28 14:37 + +### 审查输入证据 + +- Diff 范围：当前工作树相对 `fdecd5d` +- 单元测试：`python3 -m unittest discover -s tests -v`，14/14 通过 +- 语法检查：`python3 -m compileall -q src scripts/sub_one_pass tests`，通过 +- 新增文件格式：Black 与 isort，均通过 +- Diff whitespace：`git diff --check`，通过 +- dry-run 数量：smoke 20、tune 60、main 20 +- GPU 检查：尚未执行，卡已关闭 + +### 已发现并修复的问题 + +#### 🔴 validation/train 文档边界泄漏 + +- 位置：`scripts/sub_one_pass/prepare_fineweb_stream.py` +- 问题：旧实现可能把填满 validation 后的同一源文档剩余 tokens 写入 training。 +- 风险：训练集和验证集共享同一文档，validation loss 偏乐观。 +- 修复：填满 validation 后丢弃当前文档剩余 tokens，从下一篇源文档开始 training；manifest 记录丢弃数量和 split unit。 +- 回归测试：`test_fineweb_stream.py` 验证边界文档不会跨 split。 + +#### 🟡 单卡队列中断后 manifest 信息丢失 + +- 位置：`scripts/sub_one_pass/run_all.py` +- 问题：旧 runner 只在整个队列结束后写 manifest，且再次启动会覆盖同阶段历史。 +- 修复：每次状态变化原子更新；以 experiment directory upsert；保留多次 invocation 和扩展 LR 网格结果；有效 `summary.json` 默认跳过。 + +#### 🟡 任意单 LR 不足以公平比较 + +- 位置：`scripts/sub_one_pass/run_all.py` +- 问题：20 个 optimizer 各使用一个未经同预算筛选的 LR，排名会混合算法差异和调参质量。 +- 修复：增加 tune stage，统一 `0.3×/1×/3×` 网格；增加 `select_lr.py`；边界最优时返回非零状态，阻止直接进入 main。 + +### 剩余问题 + +#### 🟡 GPU 兼容性门禁待执行 + +- `soap`、`sophiag`、`d-muon` 和 `muon-pytorch` 必须在目标 GPU 环境完成 smoke。 +- `muon-pytorch` 需要所有正式实验共用的 PyTorch 环境提供 `torch.optim.Muon`。 +- 在完成 20-way smoke 前不得启动 S1 或报告预计账单时间。 + +#### 🟢 全仓格式门禁 + +- 全仓 Black/isort 检查会命中多个未由本实验修改的原始文件。 +- 本轮只验证并格式化新增实验脚本与测试，避免产生无关算法 diff。 + +### 结论 + +- 结论：`Approved`（本地设计与实现） +- 严重问题：0 个未解决 +- 建议修改：1 个外部 GPU 验证门禁 +- 可选优化：1 个全仓格式治理 +- 下一步建议：重新开卡后选择已预装兼容 PyTorch 的镜像，只运行 S0；根据 20 个 optimizer 的实测吞吐量再确认 S1/S2 总时长。 + +--- diff --git "a/docs/sub_one_pass/\346\226\271\346\241\210\350\256\276\350\256\241.md" "b/docs/sub_one_pass/\346\226\271\346\241\210\350\256\276\350\256\241.md" new file mode 100644 index 0000000..3d69506 --- /dev/null +++ "b/docs/sub_one_pass/\346\226\271\346\241\210\350\256\276\350\256\241.md" @@ -0,0 +1,117 @@ +## 2026-07-28 14:37 + +## Material Passport + +- Origin Skill: experiment-agent +- Origin Mode: plan +- Origin Date: 2026-07-28 +- Verification Status: UNVERIFIED +- Version Label: code_plan_v1 + +## Experiment Overview + +- Title: Fixed-corpus sub-one-pass LLM optimizer benchmark +- Objective: 比较仓库 20 个 optimizer 在相同训练 token 暴露下的早期样本效率。 +- Hypothesis: 不预设某个 optimizer 必然领先；检验排名是否随 token budget 与 seed 改变。 +- Type: training +- Working Directory: `/root/autodl-tmp/llm-optimizer-benchmark` +- Entry Command: `python scripts/sub_one_pass/run_all.py --stage smoke ...` +- Environment: 单张 NVIDIA A800 80GB；所有正式运行共用同一 PyTorch/CUDA 用户态软件栈。 + +## 技术方案概述 + +使用一个固定、预 tokenized 的大语料建立确定性无放回 sequence stream。所有 optimizer 共享模型初始化、数据顺序、batch、gradient accumulation、precision、warmup 和 validation batches。主实验是一条持续到 256M tokens 的轨迹，并在嵌套 token checkpoints 评估。单卡成本通过分阶段漏斗控制，而不是直接运行 20 个完整多种子实验。 + +## 研究问题与变量 + +- 研究问题：在固定大语料的 sub-one-pass 区间内，不同 optimizer 达到相同 validation quality 所需的训练 token 是否不同？ +- 自变量：optimizer。 +- 主因变量：固定 token checkpoints 的 validation cross-entropy。 +- 次因变量：log-token learning-curve AUC、tokens-to-threshold、wall-clock、tokens/s、峰值显存、失败率。 +- 控制变量：模型、初始化 seed、训练数据及顺序、validation slice、tokenizer、batch、sequence length、precision、gradient clipping、weight decay、warmup policy。 +- 潜在混杂：各 optimizer 的学习率敏感性、框架 kernel 差异、optimizer step 开销、随机 seed。 + +## 四阶段实验 + +| 阶段 | 覆盖 | 默认预算 | 用途 | 放行条件 | +|---|---:|---:|---|---| +| S0 smoke | 20 optimizers × 1 seed | 2M tokens | API、OOM、NaN、checkpoint 检查 | 20 个均完成或留下可解释失败 | +| S1 LR screening | 20 × 3 LR 倍率 × 1 seed | 16M tokens | 为每个 optimizer 选学习率区间 | 每个 optimizer 至少一个稳定配置 | +| S2 main curve | 20 × 1 seed | 16M–256M nested | 完整早期学习曲线 | 结果齐全且无数据重复 | +| S3 confirmation | AdamW + 领先组 × seeds 1,2 | 16M–256M nested | 检查排名稳定性 | paired-seed 结果齐全 | + +S1 默认倍率为 `0.3, 1.0, 3.0`，倍率同时作用于该 optimizer 的主学习率与配套学习率。若三个候选中最优值落在边界，先向该方向补一个候选，不能直接进入 S2。 + +## 固定数据协议 + +- 推荐 train corpus：1,073,741,824 tokens；validation：8,388,608 tokens。 +- 最大训练预算：268,435,456 tokens，对应约 25% 数据暴露。 +- sequence length：512；micro batch：16；gradient accumulation：8。 +- 每个 optimizer step：65,536 target tokens。 +- checkpoints：16,777,216、33,554,432、67,108,864、134,217,728、268,435,456 tokens。 +- 数据顺序由相同 `data_seed` 的确定性仿射排列生成；固定 sequence boundaries；one pass 前不重复 sequence。 +- 每次 validation 从 step 0 开始，使用完全相同的 64 个 batches。 + +## 模型与调度 + +- 模型：Llama，8 layers，6 heads，embedding width 384，约 30M 参数。 +- precision：A800 上优先 bfloat16；允许 TF32。 +- scheduler：16-step linear warmup 后 constant LR。该 schedule 不依赖最终 horizon，保证 16M checkpoint 在 256M 轨迹中有明确含义。 +- optimizer 内部 horizon：AdEMAMix 的 `alpha/beta3` warmup 在 S0/S1/S2/S3 均固定为 256M-token horizon，短阶段只截取主轨迹前缀。 +- gradient clipping：0.5；weight decay：0.1，除非算法定义要求单独处理。 + +## 公平性边界 + +固定 token 数衡量的是样本效率，不是固定 FLOPs 或固定 wall-clock。SOAP、Muon、Sophia 等 optimizer 的 step 成本不同，因此不能只用 validation loss 排名后声称“计算效率更高”。正式报告必须同时给出： + +1. validation loss vs training tokens； +2. validation loss vs wall-clock； +3. tokens/s 与峰值显存； +4. 不稳定或失败配置数量。 + +## 统计策略 + +- S2 是全算法 exploratory comparison。 +- S3 对 seed 配对：同一 seed 下各 optimizer 共享完全相同的模型初始化和数据顺序。 +- 报告每个 checkpoint 的 paired difference，而不是只报告各自均值。 +- 三个 seed 只能提供稳定性证据，不足以支撑精细正态近似；优先报告原始 seed 点、median/range，并对领先组补到 5 seeds（若计算预算允许）。 +- 主要结论预先指定为 256M-token validation loss；学习曲线 AUC 为次要结论，避免从五个 checkpoint 中事后挑最有利点。 + +## 监控配置 + +- 硬超时：只在用户确认后按阶段设置；除硬超时外不自动终止。 +- 进程监控：PID、GPU utilization、显存、`run.log` 最新更新时间。 +- 结果监控：`manifest_.json`、每个实验目录的 `summary.json`。 +- 异常策略：NaN、OOM、非零退出立即记录并通知；不静默重试。 +- 成功条件：S0 的 20 个配置均生成有效 summary，或留下可复现且经用户决定如何处理的失败。 + +## 模块划分 + +| 模块 | 职责 | +|---|---| +| `src/experiment_budget.py` | token budget、steps、data exposure 的精确约束 | +| `src/data/reader.py` | 固定边界、无放回、常数内存排列 | +| `scripts/sub_one_pass/run_all.py` | 分阶段单卡队列、LR 倍率、恢复与 manifest | +| `scripts/sub_one_pass/summarize.py` | 汇总 checkpoint 指标 | +| `scripts/sub_one_pass/select_lr.py` | 从 S1 结果生成每个 optimizer 的 LR 倍率表 | +| `scripts/sub_one_pass/estimate_runtime.py` | 用 calibration 吞吐量估算后续 GPU 时间 | + +## 关键设计决策 + +| 决策点 | 选择 | 理由 | +|---|---|---| +| pass vs token budget | 固定唯一语料，使用 exposure < 1 | 大语料上 one pass 已超出单卡预算 | +| 多预算实现 | 一条 nested trajectory | 避免重复训练，且 constant schedule 不依赖 horizon | +| 调参 | 等预算 LR screening | 单一任意 LR 会把超参数质量误当作算法质量 | +| 全算法复现 | S2 全 20；S3 只扩种子 | 保留覆盖面，同时控制单卡成本 | +| 环境 | 所有算法使用同一 PyTorch 版本 | 避免 kernel/数值栈成为 optimizer 混杂因素 | + +## 实现计划 + +1. 审计并测试 20 个 optimizer 的命令生成。 +2. 为 runner 增加 stage、LR scale、multi-seed、已完成跳过、增量 manifest。 +3. 增加 LR 选择与时间估算脚本。 +4. 完成单元测试和 CPU smoke；GPU 开卡后只做 S0 calibration。 +5. calibration 后用实测吞吐量确认 S1/S2 队列，再开始正式训练。 + +--- diff --git "a/docs/sub_one_pass/\351\234\200\346\261\202\347\220\206\350\247\243.md" "b/docs/sub_one_pass/\351\234\200\346\261\202\347\220\206\350\247\243.md" new file mode 100644 index 0000000..6d89809 --- /dev/null +++ "b/docs/sub_one_pass/\351\234\200\346\261\202\347\220\206\350\247\243.md" @@ -0,0 +1,67 @@ +## 2026-07-28 14:37 + +## 需求概述 + +在一张 GPU 上，将仓库当前暴露的 20 个 optimizer 接入统一、可恢复、可审计的 sub-one-pass 语言模型训练协议。研究对象是固定大语料上的早期样本效率，而不是 Datablations，也不是固定总计算量实验。 + +## 功能需求 + +### P0 - 必须实现 + +- [ ] 固定模型结构、tokenizer、训练/验证语料、数据顺序和 token checkpoints。 +- [ ] 训练样本在达到 one pass 前不重复，且精确记录 `tokens_seen / unique_train_tokens`。 +- [ ] 覆盖 `adamw`、`sgd`、`muon`、`soap`、`ademamix`、`lion`、`sf-adamw`、`sf-sgd`、`signsgd`、`signum`、`prodigy`、`sophiag`、`adopt`、`mars`、`adafactor`、`lamb`、`scion`、`scion-light`、`d-muon`、`muon-pytorch`。 +- [ ] 单卡串行执行，单个 optimizer 失败时记录失败并保留已完成结果。 +- [ ] 输出每个 token checkpoint 的 validation loss、perplexity、accuracy、吞吐量、训练时间和峰值显存。 +- [ ] 支持 smoke、学习率筛选、主实验和多种子确认四个阶段。 + +### P1 - 重要功能 + +- [ ] 已完成实验默认跳过，允许卡被关闭后继续队列。 +- [ ] manifest 在每个子实验结束后立即原子更新。 +- [ ] 学习率筛选对每个 optimizer 使用相同 token 预算和相同候选倍率。 +- [ ] 主实验能够读取每个 optimizer 选出的学习率倍率。 +- [ ] 结果目录和实验名包含阶段、optimizer、seed、学习率倍率和最大 token budget。 + +### P2 - 可选功能 + +- [ ] 根据 GPU calibration 结果自动估算剩余队列时间。 +- [ ] 对 paired-seed 结果输出置信区间和曲线汇总。 + +## 非功能需求 + +- 性能：默认约 30M 参数 Llama；每次只运行一个训练进程；数据排列为常数内存。 +- 可恢复性：中断后不覆盖完整结果；失败不静默重试。 +- 可复现性：保存完整命令、随机种子、数据 manifest、软件版本和 git commit。 +- 公平性：主指标按相同训练 token 数比较；wall-clock 与显存作为独立效率指标报告。 + +## 边界与约束 + +- 不包含：Datablations、数据量消融、跨模型规模结论、多卡 scaling。 +- 硬件限制：一张 GPU，正式运行前必须先 calibration。 +- 数据限制：最大训练 token budget 必须小于或等于无放回数据流的一次遍历容量。 +- 环境限制：20 个 optimizer 必须在同一 PyTorch 软件栈下比较；不允许仅为单个 optimizer 更换框架版本后直接混合排名。 + +## 验收标准 + +### 固定 token 预算 + +- Given 每步 token 数为 65,536，When 指定 16M、32M、64M、128M、256M checkpoints，Then 精确映射为 256、512、1024、2048、4096 optimizer steps。 + +### 无放回数据 + +- Given 固定 data seed 和 fixed boundaries，When 消耗小于一次遍历的数据，Then 每个训练 target sequence 最多出现一次，且不同 optimizer 看到相同顺序。 + +### 全 optimizer 覆盖 + +- Given 仓库 CLI 的 optimizer choices，When 执行 runner dry-run，Then 恰好生成 20 个唯一 optimizer 配置，无遗漏、无额外算法。 + +### 可恢复队列 + +- Given 某个实验目录已存在有效 `summary.json`，When 重启相同阶段，Then 默认跳过该实验并在 manifest 中标记 `skipped-completed`。 + +### 公平调参 + +- Given LR screening 阶段，When 比较所有 optimizer，Then 每个 optimizer 使用相同训练 token budget、相同 seed 集合和相同 LR 倍率网格。 + +--- diff --git a/exps/sub_one_pass/formal/confirm_selection.json b/exps/sub_one_pass/formal/confirm_selection.json new file mode 100644 index 0000000..57328ad --- /dev/null +++ b/exps/sub_one_pass/formal/confirm_selection.json @@ -0,0 +1,116 @@ +{ + "selection_rule": "AdamW baseline plus the three non-AdamW optimizers with the lowest S2 final validation loss at 268435456 tokens.", + "selected_optimizers": [ + "adamw", + "ademamix", + "soap", + "muon" + ], + "seeds": [ + 1, + 2 + ], + "ranking": [ + { + "optimizer": "ademamix", + "final_val_loss": 4.01993989944458, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456/summary.json" + }, + { + "optimizer": "soap", + "final_val_loss": 4.038823127746582, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456/summary.json" + }, + { + "optimizer": "muon", + "final_val_loss": 4.043658256530762, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456/summary.json" + }, + { + "optimizer": "d-muon", + "final_val_loss": 4.057147979736328, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456/summary.json" + }, + { + "optimizer": "adamw", + "final_val_loss": 4.083443641662598, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456/summary.json" + }, + { + "optimizer": "prodigy", + "final_val_loss": 4.091464042663574, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456/summary.json" + }, + { + "optimizer": "mars", + "final_val_loss": 4.1218061447143555, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456/summary.json" + }, + { + "optimizer": "lion", + "final_val_loss": 4.166975021362305, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456/summary.json" + }, + { + "optimizer": "adopt", + "final_val_loss": 4.170770168304443, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456/summary.json" + }, + { + "optimizer": "sophiag", + "final_val_loss": 4.209933280944824, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456/summary.json" + }, + { + "optimizer": "lamb", + "final_val_loss": 4.210540771484375, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456/summary.json" + }, + { + "optimizer": "signum", + "final_val_loss": 4.239973068237305, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456/summary.json" + }, + { + "optimizer": "sf-adamw", + "final_val_loss": 4.304592132568359, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456/summary.json" + }, + { + "optimizer": "adafactor", + "final_val_loss": 4.321353912353516, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456/summary.json" + }, + { + "optimizer": "muon-pytorch", + "final_val_loss": 4.4571852684021, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456/summary.json" + }, + { + "optimizer": "signsgd", + "final_val_loss": 5.291903495788574, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456/summary.json" + }, + { + "optimizer": "scion", + "final_val_loss": 6.072164535522461, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456/summary.json" + }, + { + "optimizer": "scion-light", + "final_val_loss": 6.240045070648193, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456/summary.json" + }, + { + "optimizer": "sgd", + "final_val_loss": 6.9730939865112305, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456/summary.json" + }, + { + "optimizer": "sf-sgd", + "final_val_loss": 6.992140293121338, + "summary_path": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456/summary.json" + } + ], + "created_at_unix": 1785263889.9600036 +} diff --git a/exps/sub_one_pass/formal/data_manifests/fineweb-512m-manifest.json b/exps/sub_one_pass/formal/data_manifests/fineweb-512m-manifest.json new file mode 100644 index 0000000..b08637a --- /dev/null +++ b/exps/sub_one_pass/formal/data_manifests/fineweb-512m-manifest.json @@ -0,0 +1,17 @@ +{ + "dataset": "HuggingFaceFW/fineweb", + "config": "sample-10BT", + "sources": [ + { + "path": "/root/autodl-tmp/llm-optimizer-data/fineweb-source/000_00000.parquet", + "size_bytes": 2147292183, + "sha256": "6b552ea48424648dc86d00df276f93fdfc55e9ad342ce3e4affc23a3a370792b" + } + ], + "tokenizer": "gpt2", + "train_tokens": 536870912, + "val_tokens": 8388608, + "documents_consumed": 788777, + "split_unit": "source_document", + "validation_boundary_discarded_tokens": 475 +} diff --git a/exps/sub_one_pass/formal/data_manifests/preflight-manifest.json b/exps/sub_one_pass/formal/data_manifests/preflight-manifest.json new file mode 100644 index 0000000..a14867d --- /dev/null +++ b/exps/sub_one_pass/formal/data_manifests/preflight-manifest.json @@ -0,0 +1,9 @@ +{ + "purpose": "optimizer-compatibility-preflight-only", + "synthetic": true, + "seed": 20260728, + "vocab_size": 50304, + "train_tokens": 4194304, + "val_tokens": 524288, + "eligible_for_scientific_comparison": false +} diff --git a/exps/sub_one_pass/formal/formal-confirm-pipeline.log b/exps/sub_one_pass/formal/formal-confirm-pipeline.log new file mode 100644 index 0000000..6ceb4c5 --- /dev/null +++ b/exps/sub_one_pass/formal/formal-confirm-pipeline.log @@ -0,0 +1,1215 @@ +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage confirm --optimizers adamw ademamix soap muon --seeds 1 2 --lr-scale-file /root/autodl-tmp/llm-optimizer-results/formal/selected_lr_scales.json --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[adamw seed=1 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adamw --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 1 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456 --lr 0.001 --beta1 0.8 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456', 'seed': 1, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +using fused AdamW: True + +Optimizer: +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.001 + maximize: False + weight_decay: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.001 + maximize: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.384 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.392 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.037 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.908 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.782 iter_dt=2.70e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.624 val_pp=277.027 val_acc=0.185951 +Train: Iter=300 (0.036622 effective passes) train_loss=5.412 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.606 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=5.267 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=5.065 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=5.028 iter_dt=2.71e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.092 val_pp=162.725 val_acc=0.219788 +Train: Iter=550 (0.067140 effective passes) train_loss=5.155 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.707 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=5.098 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.916 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.899 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.710 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.659 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.589 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.638 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.638 iter_dt=2.71e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.568 val_pp=96.390 val_acc=0.268028 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.684 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.489 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.505 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.432 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.448 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.498 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.518 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.519 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.373 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.459 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.375 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.641 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.210 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.557 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.241 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.261 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.564 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.166 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.115 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.297 iter_dt=2.71e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.287 val_pp=72.727 val_acc=0.291880 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.297 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.315 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.268 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.261 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.296 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.007 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.201 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.204 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.197 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.448 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.247 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.012 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.437 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.325 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.216 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.052 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.368 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.272 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.217 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.080 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.246 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.876 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.111 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.185 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.260 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.120 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.326 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.040 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.022 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.111 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.183 iter_dt=2.62e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.255 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.189 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.053 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.072 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.157 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.995 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.087 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.173 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.198 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.096 iter_dt=2.70e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.098 val_pp=60.219 val_acc=0.307306 + +[ademamix seed=1 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt ademamix --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 1 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456 --lr 0.001 --beta1 0.9 --beta2 0.999 --adema_beta3 0.999 --adema_alpha 8.0 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --adema_beta3_warmup 4096 --adema_alpha_warmup 4096 +Starting Experiment: sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456', 'seed': 1, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'ademamix', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.999, 'adema_alpha': 8.0, 'adema_beta3_warmup': 4096, 'adema_alpha_warmup': 4096, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdEMAMix ( +Parameter Group 0 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.001 + weight_decay: 0.1 + +Parameter Group 1 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.001 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.334 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.454 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.111 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.959 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.798 iter_dt=2.76e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.653 val_pp=285.078 val_acc=0.183714 +Train: Iter=300 (0.036622 effective passes) train_loss=5.404 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.587 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=5.259 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=5.030 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=4.982 iter_dt=2.73e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.039 val_pp=154.268 val_acc=0.223021 +Train: Iter=550 (0.067140 effective passes) train_loss=5.075 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.659 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=5.030 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.849 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.849 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.643 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.568 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.520 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.587 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.597 iter_dt=2.81e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.499 val_pp=89.909 val_acc=0.274397 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.621 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.416 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.441 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.362 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.382 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.419 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.423 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.449 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.301 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.360 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.307 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.571 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.133 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.497 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.167 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.190 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.503 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.099 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.046 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.235 iter_dt=2.75e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.205 val_pp=67.009 val_acc=0.300161 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.212 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.233 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.182 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.177 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.233 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.940 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.125 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.114 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.126 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.378 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.151 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.938 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.365 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.240 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.126 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.985 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.283 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.199 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.148 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=3.985 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.162 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.793 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.049 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.115 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.175 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.025 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.238 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=3.973 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.935 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.030 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.098 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.180 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.106 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=3.995 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=3.973 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.079 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.919 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.017 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.071 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.110 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.019 iter_dt=2.73e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.020 val_pp=55.683 val_acc=0.316698 + +[soap seed=1 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt soap --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 1 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456 --lr 0.001 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456', 'seed': 1, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'soap', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SOAP ( +Parameter Group 0 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.001 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.001 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.375 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.148 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=5.711 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.573 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.462 iter_dt=2.80e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.308 val_pp=202.004 val_acc=0.208868 +Train: Iter=300 (0.036622 effective passes) train_loss=5.136 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.281 iter_dt=2.87e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=4.894 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=4.646 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=4.634 iter_dt=2.80e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=4.715 val_pp=111.628 val_acc=0.256311 +Train: Iter=550 (0.067140 effective passes) train_loss=4.694 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.424 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=4.770 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.661 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.659 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.501 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.433 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.379 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.448 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.491 iter_dt=2.79e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.378 val_pp=79.661 val_acc=0.283842 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.501 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.300 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.345 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.271 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.297 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.342 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.345 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.383 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.242 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.309 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.249 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.533 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.111 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.458 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.125 iter_dt=2.88e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.142 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.444 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.062 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=3.991 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.214 iter_dt=2.80e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.176 val_pp=65.095 val_acc=0.299778 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.185 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.203 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.168 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.158 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.207 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.902 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.102 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.106 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.101 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.366 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.135 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.941 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.362 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.237 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.117 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.981 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.282 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.209 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.152 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=3.993 iter_dt=2.90e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.178 iter_dt=2.88e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.798 iter_dt=3.00e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.028 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.105 iter_dt=2.97e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.166 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.053 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.246 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=3.985 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.939 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.047 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.106 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.185 iter_dt=2.89e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.137 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.017 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=3.991 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.095 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.946 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.033 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.098 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.135 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.031 iter_dt=2.81e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.037 val_pp=56.676 val_acc=0.312168 + +[muon seed=1 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt muon --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 1 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456 --lr 0.003 --muon_lr_factor 0.03 --beta1 0.8 --beta2 0.999 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456', 'seed': 1, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 0.03, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Muon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + adamw_lr: 0.003 + adamw_lr_ratio: 0.1 + adamw_wd: 0.1 + lr: 0.03 + momentum: 0.95 + nesterov: True + ns_steps: 5 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.682 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=5.823 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=5.471 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=5.360 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=5.328 iter_dt=2.83e-01s lr=3.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.146 val_pp=171.719 val_acc=0.214575 +Train: Iter=300 (0.036622 effective passes) train_loss=4.927 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=350 (0.042725 effective passes) train_loss=5.135 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=400 (0.048829 effective passes) train_loss=4.751 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=450 (0.054932 effective passes) train_loss=4.569 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=500 (0.061036 effective passes) train_loss=4.622 iter_dt=2.83e-01s lr=3.00e-02 +>Eval: Iter=512 (0.062501 effective passes) val_loss=4.698 val_pp=109.707 val_acc=0.256454 +Train: Iter=550 (0.067140 effective passes) train_loss=4.687 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=600 (0.073243 effective passes) train_loss=4.450 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=650 (0.079347 effective passes) train_loss=4.789 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=700 (0.085451 effective passes) train_loss=4.683 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=750 (0.091554 effective passes) train_loss=4.691 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=800 (0.097658 effective passes) train_loss=4.537 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=850 (0.103761 effective passes) train_loss=4.450 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=900 (0.109865 effective passes) train_loss=4.402 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=950 (0.115969 effective passes) train_loss=4.485 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.523 iter_dt=2.84e-01s lr=3.00e-02 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.403 val_pp=81.679 val_acc=0.282814 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.534 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.342 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.365 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.306 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.303 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.358 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.374 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.382 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.262 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.324 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.278 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.552 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.121 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.461 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.132 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.177 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.463 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.068 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.010 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.203 iter_dt=2.88e-01s lr=3.00e-02 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.192 val_pp=66.174 val_acc=0.300663 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.209 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.216 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.177 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.187 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.207 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.941 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.128 iter_dt=2.82e-01s lr=3.00e-02 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.119 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.103 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.373 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.148 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.948 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.352 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.249 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.120 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.993 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.301 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.213 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.156 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.002 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.183 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.822 iter_dt=2.82e-01s lr=3.00e-02 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.057 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.109 iter_dt=2.82e-01s lr=3.00e-02 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.184 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.053 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.246 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3400 (0.415045 effective passes) train_loss=3.986 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.956 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.051 iter_dt=2.82e-01s lr=3.00e-02 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.112 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.197 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.135 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.017 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.000 iter_dt=2.82e-01s lr=3.00e-02 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.108 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.944 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.045 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.106 iter_dt=2.82e-01s lr=3.00e-02 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.146 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.039 iter_dt=2.83e-01s lr=3.00e-02 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.044 val_pp=57.038 val_acc=0.313625 + +[adamw seed=2 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adamw --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 2 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456 --lr 0.001 --beta1 0.8 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456', 'seed': 2, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +using fused AdamW: True + +Optimizer: +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.001 + maximize: False + weight_decay: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.001 + maximize: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.388 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.476 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.153 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.078 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.895 iter_dt=2.73e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.800 val_pp=330.217 val_acc=0.175360 +Train: Iter=300 (0.036622 effective passes) train_loss=5.530 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.678 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=5.407 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=5.160 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=5.063 iter_dt=2.79e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.138 val_pp=170.332 val_acc=0.215649 +Train: Iter=550 (0.067140 effective passes) train_loss=5.192 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.750 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=5.113 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.949 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.926 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.754 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.681 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.612 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.668 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.674 iter_dt=2.79e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.594 val_pp=98.895 val_acc=0.267216 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.725 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.511 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.518 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.450 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.485 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.527 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.521 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.542 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.396 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.474 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.396 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.678 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.224 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.583 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.260 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.291 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.594 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.190 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.146 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.335 iter_dt=2.71e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.306 val_pp=74.144 val_acc=0.290478 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.314 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.333 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.284 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.266 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.319 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.035 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.215 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.218 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.199 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.476 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.248 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.044 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.453 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.330 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.223 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.075 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.375 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.296 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.233 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.076 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.254 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.886 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.132 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.192 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.264 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.137 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.329 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.059 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.029 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.121 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.199 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.265 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.210 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.074 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.079 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.192 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.009 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.107 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.181 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.209 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.093 iter_dt=2.76e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.109 val_pp=60.879 val_acc=0.306969 + +[ademamix seed=2 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt ademamix --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 2 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456 --lr 0.001 --beta1 0.9 --beta2 0.999 --adema_beta3 0.999 --adema_alpha 8.0 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --adema_beta3_warmup 4096 --adema_alpha_warmup 4096 +Starting Experiment: sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456', 'seed': 2, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'ademamix', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.999, 'adema_alpha': 8.0, 'adema_beta3_warmup': 4096, 'adema_alpha_warmup': 4096, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdEMAMix ( +Parameter Group 0 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.001 + weight_decay: 0.1 + +Parameter Group 1 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.001 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.357 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.503 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.109 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.980 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.803 iter_dt=2.74e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.665 val_pp=288.511 val_acc=0.182850 +Train: Iter=300 (0.036622 effective passes) train_loss=5.389 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.579 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=5.230 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=4.993 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=4.895 iter_dt=2.76e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=4.976 val_pp=144.854 val_acc=0.231894 +Train: Iter=550 (0.067140 effective passes) train_loss=4.982 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.615 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=4.969 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.818 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.819 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.633 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.536 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.493 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.566 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.589 iter_dt=2.74e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.477 val_pp=88.002 val_acc=0.276651 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.584 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.404 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.424 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.337 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.363 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.414 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.419 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.424 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.289 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.373 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.297 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.573 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.131 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.478 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.151 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.181 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.488 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.085 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.010 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.218 iter_dt=2.73e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.196 val_pp=66.403 val_acc=0.300983 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.210 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.234 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.182 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.171 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.219 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.931 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.119 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.121 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.129 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.360 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.140 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.940 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.351 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.230 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.133 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.989 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.274 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.192 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.141 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.002 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.159 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.783 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.039 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.096 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.169 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.036 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.252 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=3.955 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.926 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.026 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.098 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.155 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.109 iter_dt=2.90e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=3.990 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=3.971 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.089 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.914 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.021 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.063 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.108 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.012 iter_dt=2.74e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.015 val_pp=55.441 val_acc=0.316336 + +[soap seed=2 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt soap --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 2 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456 --lr 0.001 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456', 'seed': 2, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'soap', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SOAP ( +Parameter Group 0 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.001 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.001 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.438 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.148 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=5.693 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.573 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.449 iter_dt=2.84e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.292 val_pp=198.685 val_acc=0.210087 +Train: Iter=300 (0.036622 effective passes) train_loss=5.132 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.280 iter_dt=2.87e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=4.871 iter_dt=2.89e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=4.619 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=4.631 iter_dt=2.84e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=4.710 val_pp=111.081 val_acc=0.257057 +Train: Iter=550 (0.067140 effective passes) train_loss=4.671 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.424 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=4.764 iter_dt=2.86e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.663 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.676 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.492 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.422 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.383 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.446 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.483 iter_dt=2.83e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.380 val_pp=79.876 val_acc=0.283529 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.506 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.300 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.343 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.276 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.299 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.342 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.366 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.383 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.228 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.319 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.257 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.537 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.106 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.449 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.141 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.170 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.461 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.073 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=3.996 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.208 iter_dt=2.80e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.179 val_pp=65.316 val_acc=0.300276 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.195 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.195 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.156 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.165 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.212 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.919 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.119 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.106 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.115 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.360 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.150 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.943 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.357 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.229 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.125 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.976 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.283 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.190 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.146 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.000 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.187 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.805 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.038 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.102 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.170 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.056 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.270 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=3.978 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.955 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.033 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.109 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.200 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.125 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.016 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=3.987 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.098 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.939 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.035 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.102 iter_dt=2.95e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.143 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.031 iter_dt=2.80e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.037 val_pp=56.663 val_acc=0.312256 + +[muon seed=2 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt muon --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 2 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456 --lr 0.003 --muon_lr_factor 0.03 --beta1 0.8 --beta2 0.999 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456', 'seed': 2, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 0.03, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Muon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + adamw_lr: 0.003 + adamw_lr_ratio: 0.1 + adamw_wd: 0.1 + lr: 0.03 + momentum: 0.95 + nesterov: True + ns_steps: 5 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.661 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=5.798 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=5.455 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=5.351 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=5.319 iter_dt=2.90e-01s lr=3.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.148 val_pp=172.119 val_acc=0.214087 +Train: Iter=300 (0.036622 effective passes) train_loss=4.954 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=350 (0.042725 effective passes) train_loss=5.123 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=400 (0.048829 effective passes) train_loss=4.758 iter_dt=2.91e-01s lr=3.00e-02 +Train: Iter=450 (0.054932 effective passes) train_loss=4.578 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=500 (0.061036 effective passes) train_loss=4.608 iter_dt=2.86e-01s lr=3.00e-02 +>Eval: Iter=512 (0.062501 effective passes) val_loss=4.698 val_pp=109.759 val_acc=0.256956 +Train: Iter=550 (0.067140 effective passes) train_loss=4.688 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=600 (0.073243 effective passes) train_loss=4.434 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=650 (0.079347 effective passes) train_loss=4.785 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=700 (0.085451 effective passes) train_loss=4.679 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=750 (0.091554 effective passes) train_loss=4.682 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=800 (0.097658 effective passes) train_loss=4.516 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=850 (0.103761 effective passes) train_loss=4.439 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=900 (0.109865 effective passes) train_loss=4.396 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=950 (0.115969 effective passes) train_loss=4.472 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.503 iter_dt=2.89e-01s lr=3.00e-02 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.398 val_pp=81.320 val_acc=0.282288 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.515 iter_dt=2.91e-01s lr=3.00e-02 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.335 iter_dt=2.91e-01s lr=3.00e-02 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.370 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.285 iter_dt=2.91e-01s lr=3.00e-02 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.318 iter_dt=2.99e-01s lr=3.00e-02 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.357 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.364 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.386 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.257 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.329 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.276 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.545 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.105 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.445 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.148 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.160 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.463 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.060 iter_dt=2.72e-01s lr=3.00e-02 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.015 iter_dt=2.91e-01s lr=3.00e-02 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.206 iter_dt=2.94e-01s lr=3.00e-02 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.190 val_pp=66.034 val_acc=0.300352 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.203 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.215 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.193 iter_dt=2.92e-01s lr=3.00e-02 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.187 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.210 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.916 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.129 iter_dt=2.92e-01s lr=3.00e-02 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.123 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.106 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.374 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.155 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.957 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.366 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.259 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.132 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.995 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.299 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.208 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.156 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=3000 (0.366217 effective passes) train_loss=3.998 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.189 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.806 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.029 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.116 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.181 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.044 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.259 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3400 (0.415045 effective passes) train_loss=3.990 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.952 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.056 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.126 iter_dt=2.91e-01s lr=3.00e-02 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.185 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.148 iter_dt=2.91e-01s lr=3.00e-02 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.032 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.007 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.120 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.956 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.045 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.117 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.125 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.046 iter_dt=2.86e-01s lr=3.00e-02 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.044 val_pp=57.070 val_acc=0.313566 +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/summarize.py /root/autodl-tmp/llm-optimizer-results/formal +Wrote 222 rows to /root/autodl-tmp/llm-optimizer-results/formal/sub_one_pass_results.csv diff --git a/exps/sub_one_pass/formal/formal-pipeline.log b/exps/sub_one_pass/formal/formal-pipeline.log new file mode 100644 index 0000000..f10042c --- /dev/null +++ b/exps/sub_one_pass/formal/formal-pipeline.log @@ -0,0 +1,9040 @@ +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[adamw seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adamw --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.8 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +using fused AdamW: True + +Optimizer: +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.0003 + maximize: False + weight_decay: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.0003 + maximize: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.330 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.384 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.082 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.069 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=5.924 iter_dt=2.71e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.826 val_pp=338.991 val_acc=0.187662 + +[adamw seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adamw --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.8 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +using fused AdamW: True + +Optimizer: +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.001 + maximize: False + weight_decay: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.001 + maximize: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.346 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.392 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.058 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.896 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.759 iter_dt=2.75e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.634 val_pp=279.670 val_acc=0.185808 + +[adamw seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adamw --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.8 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +using fused AdamW: True + +Optimizer: +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.003 + maximize: False + weight_decay: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.003 + maximize: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.736 iter_dt=2.74e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.974 iter_dt=2.68e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.761 iter_dt=2.70e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.556 iter_dt=2.71e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.427 iter_dt=2.80e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.316 val_pp=553.568 val_acc=0.140190 + +[sgd seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216 --lr 0.003 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGD ( +Parameter Group 0 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.003 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.003 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=9.957 iter_dt=2.77e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=9.120 iter_dt=2.75e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=8.751 iter_dt=2.76e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=8.698 iter_dt=2.79e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=8.428 iter_dt=2.75e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=8.424 val_pp=4556.842 val_acc=0.058386 + +[sgd seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216 --lr 0.01 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.01, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGD ( +Parameter Group 0 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.01 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.01 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=9.138 iter_dt=2.74e-01s lr=1.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=8.395 iter_dt=2.73e-01s lr=1.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=8.102 iter_dt=2.69e-01s lr=1.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=8.111 iter_dt=2.73e-01s lr=1.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=7.845 iter_dt=2.73e-01s lr=1.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.847 val_pp=2558.660 val_acc=0.078848 + +[sgd seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216 --lr 0.03 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.03, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGD ( +Parameter Group 0 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.03 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.03 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.677 iter_dt=2.70e-01s lr=3.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=7.972 iter_dt=2.70e-01s lr=3.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=7.785 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=7.890 iter_dt=2.76e-01s lr=3.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=7.716 iter_dt=2.73e-01s lr=3.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.710 val_pp=2230.875 val_acc=0.053326 + +[muon seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt muon --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --muon_lr_factor 0.003 --beta1 0.8 --beta2 0.999 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 0.003, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Muon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + adamw_lr: 0.0003 + adamw_lr_ratio: 0.09999999999999999 + adamw_wd: 0.1 + lr: 0.003 + momentum: 0.95 + nesterov: True + ns_steps: 5 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.384 iter_dt=2.91e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.422 iter_dt=2.92e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.163 iter_dt=2.92e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.094 iter_dt=2.96e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.967 iter_dt=2.93e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.848 val_pp=346.491 val_acc=0.183538 + +[muon seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt muon --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216 --lr 0.001 --muon_lr_factor 0.01 --beta1 0.8 --beta2 0.999 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 0.01, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Muon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + adamw_lr: 0.001 + adamw_lr_ratio: 0.1 + adamw_wd: 0.1 + lr: 0.01 + momentum: 0.95 + nesterov: True + ns_steps: 5 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.762 iter_dt=2.89e-01s lr=1.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=6.032 iter_dt=2.88e-01s lr=1.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=5.684 iter_dt=2.86e-01s lr=1.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=5.522 iter_dt=2.87e-01s lr=1.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=5.414 iter_dt=2.88e-01s lr=1.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.261 val_pp=192.755 val_acc=0.210743 + +[muon seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt muon --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216 --lr 0.003 --muon_lr_factor 0.03 --beta1 0.8 --beta2 0.999 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 0.03, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Muon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + adamw_lr: 0.003 + adamw_lr_ratio: 0.1 + adamw_wd: 0.1 + lr: 0.03 + momentum: 0.95 + nesterov: True + ns_steps: 5 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.665 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=5.831 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=5.494 iter_dt=2.95e-01s lr=3.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=5.375 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=5.330 iter_dt=2.87e-01s lr=3.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.157 val_pp=173.725 val_acc=0.213518 + +[soap seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt soap --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'soap', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SOAP ( +Parameter Group 0 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.0003 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.0003 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.838 iter_dt=2.78e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=7.767 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=7.085 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.685 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.351 iter_dt=2.87e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.225 val_pp=505.345 val_acc=0.183193 + +[soap seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt soap --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'soap', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SOAP ( +Parameter Group 0 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.001 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.001 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.381 iter_dt=2.97e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.149 iter_dt=2.91e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=5.709 iter_dt=2.89e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.569 iter_dt=2.86e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.440 iter_dt=2.86e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.292 val_pp=198.724 val_acc=0.209654 + +[soap seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt soap --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'soap', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SOAP ( +Parameter Group 0 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.003 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.003 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.442 iter_dt=2.83e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.490 iter_dt=2.84e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.093 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.905 iter_dt=2.85e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.732 iter_dt=2.83e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.588 val_pp=267.297 val_acc=0.185127 + +[ademamix seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt ademamix --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.9 --beta2 0.999 --adema_beta3 0.999 --adema_alpha 8.0 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --adema_beta3_warmup 4096 --adema_alpha_warmup 4096 +Starting Experiment: sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'ademamix', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.999, 'adema_alpha': 8.0, 'adema_beta3_warmup': 4096, 'adema_alpha_warmup': 4096, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdEMAMix ( +Parameter Group 0 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.0003 + weight_decay: 0.1 + +Parameter Group 1 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.0003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.476 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.474 iter_dt=2.77e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.130 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.066 iter_dt=2.81e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=5.898 iter_dt=2.80e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.787 val_pp=326.190 val_acc=0.186926 + +[ademamix seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt ademamix --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.9 --beta2 0.999 --adema_beta3 0.999 --adema_alpha 8.0 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --adema_beta3_warmup 4096 --adema_alpha_warmup 4096 +Starting Experiment: sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'ademamix', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.999, 'adema_alpha': 8.0, 'adema_beta3_warmup': 4096, 'adema_alpha_warmup': 4096, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdEMAMix ( +Parameter Group 0 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.001 + weight_decay: 0.1 + +Parameter Group 1 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.001 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.349 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.449 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.112 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.960 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.798 iter_dt=2.78e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.665 val_pp=288.530 val_acc=0.183367 + +[ademamix seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt ademamix --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.9 --beta2 0.999 --adema_beta3 0.999 --adema_alpha 8.0 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --adema_beta3_warmup 4096 --adema_alpha_warmup 4096 +Starting Experiment: sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'ademamix', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.999, 'adema_alpha': 8.0, 'adema_beta3_warmup': 4096, 'adema_alpha_warmup': 4096, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdEMAMix ( +Parameter Group 0 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.003 + weight_decay: 0.1 + +Parameter Group 1 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.719 iter_dt=2.92e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.078 iter_dt=2.79e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.763 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.680 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.531 iter_dt=2.77e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.462 val_pp=640.196 val_acc=0.128218 + +[lion seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt lion --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.9 --beta2 0.99 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.99, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lion ( +Parameter Group 0 + betas: (0.9, 0.99) + lr: 0.0003 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.99) + lr: 0.0003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.614 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.893 iter_dt=2.76e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.630 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.554 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.389 iter_dt=2.74e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.277 val_pp=531.994 val_acc=0.146252 + +[lion seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt lion --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.9 --beta2 0.99 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.99, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lion ( +Parameter Group 0 + betas: (0.9, 0.99) + lr: 0.001 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.99) + lr: 0.001 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.579 iter_dt=2.95e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.019 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.878 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.972 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.916 iter_dt=2.76e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.849 val_pp=942.977 val_acc=0.099554 + +[lion seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt lion --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.9 --beta2 0.99 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.99, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lion ( +Parameter Group 0 + betas: (0.9, 0.99) + lr: 0.003 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.99) + lr: 0.003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.017 iter_dt=2.71e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.599 iter_dt=2.73e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=7.506 iter_dt=2.73e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=7.631 iter_dt=2.71e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=7.413 iter_dt=2.72e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.386 val_pp=1613.586 val_acc=0.071882 + +[sf-adamw seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sf-adamw --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.9 --beta2 0.9999 --scheduler none --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.9999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdamWScheduleFree ( +Parameter Group 0 + betas: (0.9, 0.9999) + eps: 1e-08 + foreach: True + k: 0 + lr: 0.0003 + lr_max: -1.0 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + betas: (0.9, 0.9999) + eps: 1e-08 + foreach: True + k: 0 + lr: 0.0003 + lr_max: -1.0 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.649 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.686 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.388 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.382 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.255 iter_dt=2.75e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.212 val_pp=498.683 val_acc=0.170010 + +[sf-adamw seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sf-adamw --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.9 --beta2 0.9999 --scheduler none --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.9999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdamWScheduleFree ( +Parameter Group 0 + betas: (0.9, 0.9999) + eps: 1e-08 + foreach: True + k: 0 + lr: 0.001 + lr_max: -1.0 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + betas: (0.9, 0.9999) + eps: 1e-08 + foreach: True + k: 0 + lr: 0.001 + lr_max: -1.0 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.192 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.432 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.210 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.219 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.101 iter_dt=2.70e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.036 val_pp=418.182 val_acc=0.167719 + +[sf-adamw seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sf-adamw --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.9 --beta2 0.9999 --scheduler none --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.9999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdamWScheduleFree ( +Parameter Group 0 + betas: (0.9, 0.9999) + eps: 1e-08 + foreach: True + k: 0 + lr: 0.003 + lr_max: -1.0 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + betas: (0.9, 0.9999) + eps: 1e-08 + foreach: True + k: 0 + lr: 0.003 + lr_max: -1.0 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.357 iter_dt=2.71e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.801 iter_dt=2.79e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.580 iter_dt=2.72e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.577 iter_dt=2.72e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.477 iter_dt=2.72e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.418 val_pp=612.911 val_acc=0.132021 + +[sf-sgd seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sf-sgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216 --lr 0.003 --momentum 0.9 --scheduler none --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGDScheduleFree ( +Parameter Group 0 + foreach: True + k: 0 + lr: 0.003 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + foreach: True + k: 0 + lr: 0.003 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=10.781 iter_dt=2.70e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=10.585 iter_dt=2.70e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=10.452 iter_dt=2.71e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=10.401 iter_dt=2.71e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=10.309 iter_dt=2.73e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=10.325 val_pp=30494.943 val_acc=0.034683 + +[sf-sgd seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sf-sgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216 --lr 0.01 --momentum 0.9 --scheduler none --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.01, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGDScheduleFree ( +Parameter Group 0 + foreach: True + k: 0 + lr: 0.01 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + foreach: True + k: 0 + lr: 0.01 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=10.493 iter_dt=2.69e-01s lr=1.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=10.197 iter_dt=2.67e-01s lr=1.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=9.989 iter_dt=2.67e-01s lr=1.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=9.878 iter_dt=2.68e-01s lr=1.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=9.660 iter_dt=2.66e-01s lr=1.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=9.712 val_pp=16513.055 val_acc=0.037819 + +[sf-sgd seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sf-sgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216 --lr 0.03 --momentum 0.9 --scheduler none --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.03, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGDScheduleFree ( +Parameter Group 0 + foreach: True + k: 0 + lr: 0.03 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + foreach: True + k: 0 + lr: 0.03 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=10.092 iter_dt=2.69e-01s lr=3.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=9.505 iter_dt=2.69e-01s lr=3.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=9.150 iter_dt=2.69e-01s lr=3.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=9.079 iter_dt=2.70e-01s lr=3.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=8.863 iter_dt=2.69e-01s lr=3.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=8.906 val_pp=7377.391 val_acc=0.039589 + +[signsgd seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt signsgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'signsgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Signum ( +Parameter Group 0 + dampening: 0.0 + lr: 0.0003 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0.0 + lr: 0.0003 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.497 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.773 iter_dt=2.68e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.607 iter_dt=2.62e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.665 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.565 iter_dt=2.67e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.476 val_pp=649.133 val_acc=0.151705 + +[signsgd seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt signsgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216 --lr 0.001 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'signsgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Signum ( +Parameter Group 0 + dampening: 0.0 + lr: 0.001 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0.0 + lr: 0.001 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.822 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.048 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.855 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.814 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.840 iter_dt=2.69e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.683 val_pp=798.798 val_acc=0.133631 + +[signsgd seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt signsgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216 --lr 0.003 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'signsgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Signum ( +Parameter Group 0 + dampening: 0.0 + lr: 0.003 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0.0 + lr: 0.003 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.846 iter_dt=2.71e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.349 iter_dt=2.71e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=7.010 iter_dt=2.70e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=7.113 iter_dt=2.83e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.960 iter_dt=2.73e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.855 val_pp=948.329 val_acc=0.114681 + +[signum seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt signum --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'signum', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Signum ( +Parameter Group 0 + dampening: 0.0 + lr: 0.0003 + momentum: 0.9 + nesterov: False + sign_update: True + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0.0 + lr: 0.0003 + momentum: 0.9 + nesterov: False + sign_update: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.219 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=7.310 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.811 iter_dt=2.77e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.641 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.399 iter_dt=2.73e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.296 val_pp=542.439 val_acc=0.146202 + +[signum seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt signum --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216 --lr 0.001 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'signum', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Signum ( +Parameter Group 0 + dampening: 0.0 + lr: 0.001 + momentum: 0.9 + nesterov: False + sign_update: True + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0.0 + lr: 0.001 + momentum: 0.9 + nesterov: False + sign_update: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.903 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.397 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.743 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.568 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.318 iter_dt=2.80e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.220 val_pp=502.641 val_acc=0.149128 + +[signum seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt signum --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216 --lr 0.003 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'signum', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Signum ( +Parameter Group 0 + dampening: 0.0 + lr: 0.003 + momentum: 0.9 + nesterov: False + sign_update: True + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0.0 + lr: 0.003 + momentum: 0.9 + nesterov: False + sign_update: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.135 iter_dt=2.69e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.197 iter_dt=2.70e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.662 iter_dt=2.70e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.599 iter_dt=2.69e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.466 iter_dt=2.70e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.385 val_pp=592.681 val_acc=0.139008 + +[prodigy seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt prodigy --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216 --lr 0.3 --beta1 0.9 --beta2 0.999 --prodigy_use_bias_correction True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.3, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'prodigy', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': True, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Using decoupled weight decay + +Optimizer: +Prodigy ( +Parameter Group 0 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 0.3 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.1 + +Parameter Group 1 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 0.3 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=10.238 iter_dt=2.84e-01s lr=3.00e-01 +effective_lr=6.14e-06 +Train: Iter=100 (0.012207 effective passes) train_loss=7.053 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=1.58e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.421 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=1.90e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.258 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=2.45e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.011 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=3.60e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.900 val_pp=364.954 val_acc=0.172651 + +[prodigy seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt prodigy --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216 --lr 1 --beta1 0.9 --beta2 0.999 --prodigy_use_bias_correction True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 1.0, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'prodigy', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': True, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Using decoupled weight decay + +Optimizer: +Prodigy ( +Parameter Group 0 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 1.0 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.1 + +Parameter Group 1 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 1.0 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.590 iter_dt=2.82e-01s lr=1.00e+00 +effective_lr=2.61e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=7.034 iter_dt=2.84e-01s lr=1.00e+00 +effective_lr=3.61e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.678 iter_dt=2.82e-01s lr=1.00e+00 +effective_lr=4.36e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.577 iter_dt=2.83e-01s lr=1.00e+00 +effective_lr=4.96e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.417 iter_dt=2.82e-01s lr=1.00e+00 +effective_lr=5.48e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.313 val_pp=551.823 val_acc=0.142332 + +[prodigy seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt prodigy --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216 --lr 3 --beta1 0.9 --beta2 0.999 --prodigy_use_bias_correction True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 3.0, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'prodigy', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': True, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Using decoupled weight decay + +Optimizer: +Prodigy ( +Parameter Group 0 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 3.0 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.1 + +Parameter Group 1 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 3.0 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.813 iter_dt=2.79e-01s lr=3.00e+00 +effective_lr=5.69e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=7.152 iter_dt=2.77e-01s lr=3.00e+00 +effective_lr=7.87e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.765 iter_dt=2.77e-01s lr=3.00e+00 +effective_lr=9.50e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.645 iter_dt=2.77e-01s lr=3.00e+00 +effective_lr=1.08e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.469 iter_dt=2.77e-01s lr=3.00e+00 +effective_lr=1.20e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.387 val_pp=593.959 val_acc=0.135414 + +[sophiag seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sophiag --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sophiag', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SophiaG ( +Parameter Group 0 + betas: (0.9, 0.999) + capturable: False + lr: 0.0003 + maximize: False + rho: 0.04 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + capturable: False + lr: 0.0003 + maximize: False + rho: 0.04 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.701 iter_dt=3.24e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.784 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.428 iter_dt=3.22e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.336 iter_dt=3.22e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.145 iter_dt=3.22e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.051 val_pp=424.420 val_acc=0.160858 + +[sophiag seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sophiag --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sophiag', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SophiaG ( +Parameter Group 0 + betas: (0.9, 0.999) + capturable: False + lr: 0.001 + maximize: False + rho: 0.04 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + capturable: False + lr: 0.001 + maximize: False + rho: 0.04 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.601 iter_dt=3.21e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.793 iter_dt=3.22e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.489 iter_dt=3.24e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.454 iter_dt=3.23e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.287 iter_dt=3.23e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.201 val_pp=493.297 val_acc=0.148426 + +[sophiag seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sophiag --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sophiag', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SophiaG ( +Parameter Group 0 + betas: (0.9, 0.999) + capturable: False + lr: 0.003 + maximize: False + rho: 0.04 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + capturable: False + lr: 0.003 + maximize: False + rho: 0.04 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.333 iter_dt=3.17e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.600 iter_dt=3.18e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=7.281 iter_dt=3.18e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=7.218 iter_dt=3.20e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=7.025 iter_dt=3.19e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.951 val_pp=1044.170 val_acc=0.098793 + +[adopt seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adopt --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adopt', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +ADOPT ( +Parameter Group 0 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.0003 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.0003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.364 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.555 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.231 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.158 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=5.976 iter_dt=2.71e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.876 val_pp=356.393 val_acc=0.179474 + +[adopt seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adopt --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adopt', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +ADOPT ( +Parameter Group 0 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.001 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.001 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.524 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.760 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.420 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.352 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.198 iter_dt=2.73e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.104 val_pp=447.420 val_acc=0.158995 + +[adopt seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adopt --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adopt', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +ADOPT ( +Parameter Group 0 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.003 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.546 iter_dt=2.72e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.836 iter_dt=2.73e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.568 iter_dt=2.73e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.583 iter_dt=2.75e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.451 iter_dt=2.79e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.370 val_pp=584.002 val_acc=0.136562 + +[mars seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt mars --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --mars_lr 0.0009 --beta1 0.8 --beta2 0.999 --mars_beta1 0.95 --mars_beta2 0.99 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'mars', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.0009, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +MARS ( +Parameter Group 0 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.0009 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.1 + weight_decay_1d: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.0009 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.0 + weight_decay_1d: 0.1 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.226 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.450 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.169 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.108 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=5.946 iter_dt=2.82e-01s lr=9.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.844 val_pp=345.245 val_acc=0.178284 + +[mars seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt mars --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216 --lr 0.001 --mars_lr 0.003 --beta1 0.8 --beta2 0.999 --mars_beta1 0.95 --mars_beta2 0.99 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'mars', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +MARS ( +Parameter Group 0 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.003 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.1 + weight_decay_1d: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.003 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.0 + weight_decay_1d: 0.1 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.338 iter_dt=2.78e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.713 iter_dt=2.77e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.482 iter_dt=2.77e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.435 iter_dt=2.78e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.293 iter_dt=2.77e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.187 val_pp=486.185 val_acc=0.151686 + +[mars seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt mars --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216 --lr 0.003 --mars_lr 0.009 --beta1 0.8 --beta2 0.999 --mars_beta1 0.95 --mars_beta2 0.99 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'mars', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.009, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +MARS ( +Parameter Group 0 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.009 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.1 + weight_decay_1d: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.009 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.0 + weight_decay_1d: 0.1 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.712 iter_dt=2.78e-01s lr=9.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.038 iter_dt=2.78e-01s lr=9.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.808 iter_dt=2.79e-01s lr=9.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.794 iter_dt=2.78e-01s lr=9.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.670 iter_dt=2.79e-01s lr=9.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.590 val_pp=727.953 val_acc=0.120892 + +[adafactor seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adafactor --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adafactor', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Adafactor ( +Parameter Group 0 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.0003 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.1 + +Parameter Group 1 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.0003 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.464 iter_dt=2.91e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.465 iter_dt=2.92e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.161 iter_dt=2.93e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.141 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=5.998 iter_dt=2.88e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.907 val_pp=367.467 val_acc=0.181429 + +[adafactor seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adafactor --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adafactor', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Adafactor ( +Parameter Group 0 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.001 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.1 + +Parameter Group 1 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.001 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.464 iter_dt=2.87e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.465 iter_dt=2.89e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.159 iter_dt=2.88e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.138 iter_dt=2.88e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.000 iter_dt=2.89e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.910 val_pp=368.706 val_acc=0.181269 + +[adafactor seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adafactor --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adafactor', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Adafactor ( +Parameter Group 0 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.003 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.1 + +Parameter Group 1 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.003 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.464 iter_dt=2.90e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.465 iter_dt=2.89e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.160 iter_dt=2.89e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.141 iter_dt=2.89e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.000 iter_dt=2.90e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.909 val_pp=368.475 val_acc=0.181158 + +[lamb seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt lamb --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lamb', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lamb ( +Parameter Group 0 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.0003 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.0003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=10.391 iter_dt=2.79e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=10.190 iter_dt=2.80e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=10.073 iter_dt=2.79e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=10.000 iter_dt=2.80e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=9.877 iter_dt=2.82e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=9.872 val_pp=19389.059 val_acc=0.088730 + +[lamb seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt lamb --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lamb', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lamb ( +Parameter Group 0 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.001 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.001 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=10.191 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=9.783 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=9.579 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=9.445 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=9.274 iter_dt=2.84e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=9.245 val_pp=10351.511 val_acc=0.131166 + +[lamb seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt lamb --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lamb', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lamb ( +Parameter Group 0 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.003 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=9.845 iter_dt=2.91e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=9.185 iter_dt=2.88e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=8.729 iter_dt=2.90e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=8.418 iter_dt=2.91e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=8.025 iter_dt=2.94e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.944 val_pp=2819.426 val_acc=0.156244 + +[scion seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt scion --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +Scion ( +Parameter Group 0 + lr: 0.0003 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.0003 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.070 iter_dt=2.81e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.437 iter_dt=2.81e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.296 iter_dt=2.81e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.345 iter_dt=2.82e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.367 iter_dt=2.83e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.255 val_pp=520.485 val_acc=0.140451 + +[scion seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt scion --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216 --lr 0.001 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +Scion ( +Parameter Group 0 + lr: 0.001 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.001 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.988 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.473 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.374 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.358 iter_dt=2.86e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.412 iter_dt=2.86e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.320 val_pp=555.530 val_acc=0.121758 + +[scion seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt scion --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216 --lr 0.003 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +Scion ( +Parameter Group 0 + lr: 0.003 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.003 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=9.169 iter_dt=2.85e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.972 iter_dt=2.85e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.762 iter_dt=2.86e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.747 iter_dt=2.84e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.572 iter_dt=2.85e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.582 val_pp=721.782 val_acc=0.113983 + +[scion-light seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt scion-light --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion-light', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +ScionLight ( +Parameter Group 0 + lr: 0.0003 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.0003 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.336 iter_dt=2.93e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.691 iter_dt=2.79e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.536 iter_dt=2.79e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.580 iter_dt=2.79e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.600 iter_dt=2.80e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.454 val_pp=635.227 val_acc=0.142559 + +[scion-light seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt scion-light --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216 --lr 0.001 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion-light', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +ScionLight ( +Parameter Group 0 + lr: 0.001 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.001 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.197 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.647 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.505 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.542 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.537 iter_dt=2.84e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.479 val_pp=651.508 val_acc=0.115211 + +[scion-light seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt scion-light --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216 --lr 0.003 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion-light', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +ScionLight ( +Parameter Group 0 + lr: 0.003 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.003 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.870 iter_dt=2.84e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.960 iter_dt=2.84e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.727 iter_dt=2.85e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.858 iter_dt=2.86e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.754 iter_dt=2.87e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.810 val_pp=907.229 val_acc=0.103092 + +[d-muon seed=0 lr_scale=0.3] /root/miniconda3/bin/torchrun --standalone --nproc_per_node=1 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --distributed_backend nccl --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt d-muon --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.8 --beta2 0.999 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': 'nccl', 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'd-muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Distributed DataReader Initialized for Worker 0/1 +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +DistributedMuon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.0003 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.1 + +Parameter Group 1 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.0003 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.481 iter_dt=2.96e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.495 iter_dt=2.81e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.233 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.209 iter_dt=2.82e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.126 iter_dt=2.80e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.031 val_pp=415.935 val_acc=0.161667 + +[d-muon seed=0 lr_scale=1] /root/miniconda3/bin/torchrun --standalone --nproc_per_node=1 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --distributed_backend nccl --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt d-muon --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.8 --beta2 0.999 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': 'nccl', 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'd-muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Distributed DataReader Initialized for Worker 0/1 +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +DistributedMuon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.001 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.1 + +Parameter Group 1 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.001 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.782 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.065 iter_dt=2.90e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=5.830 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.658 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.561 iter_dt=2.81e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.397 val_pp=220.810 val_acc=0.199068 + +[d-muon seed=0 lr_scale=3] /root/miniconda3/bin/torchrun --standalone --nproc_per_node=1 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --distributed_backend nccl --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt d-muon --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.8 --beta2 0.999 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': 'nccl', 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'd-muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Distributed DataReader Initialized for Worker 0/1 +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +DistributedMuon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.003 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.1 + +Parameter Group 1 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.003 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.577 iter_dt=2.83e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=5.880 iter_dt=2.84e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=5.535 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.348 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.313 iter_dt=2.83e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.128 val_pp=168.674 val_acc=0.215315 + +[muon-pytorch seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt muon-pytorch --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216 --lr 0.006 --muon_adamw_lr 0.0003 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.006, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon-pytorch', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.0003, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: + +Train: Iter=50 (0.006104 effective passes) train_loss=7.390 iter_dt=2.77e-01s lr=6.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.445 iter_dt=2.77e-01s lr=6.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.154 iter_dt=2.77e-01s lr=6.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.076 iter_dt=2.77e-01s lr=6.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.913 iter_dt=2.78e-01s lr=6.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.794 val_pp=328.169 val_acc=0.192841 + +[muon-pytorch seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt muon-pytorch --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216 --lr 0.02 --muon_adamw_lr 0.001 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.02, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon-pytorch', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: + +Train: Iter=50 (0.006104 effective passes) train_loss=6.818 iter_dt=2.78e-01s lr=2.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=6.057 iter_dt=2.78e-01s lr=2.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=5.715 iter_dt=2.77e-01s lr=2.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=5.620 iter_dt=2.77e-01s lr=2.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=5.505 iter_dt=2.78e-01s lr=2.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.356 val_pp=211.939 val_acc=0.205448 + +[muon-pytorch seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt muon-pytorch --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216 --lr 0.06 --muon_adamw_lr 0.003 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.06, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon-pytorch', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.003, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: + +Train: Iter=50 (0.006104 effective passes) train_loss=6.799 iter_dt=2.77e-01s lr=6.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=5.850 iter_dt=2.77e-01s lr=6.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=5.538 iter_dt=2.76e-01s lr=6.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=5.469 iter_dt=2.76e-01s lr=6.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=5.429 iter_dt=2.77e-01s lr=6.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.283 val_pp=197.032 val_acc=0.205511 +Wrote selected scales to /root/autodl-tmp/llm-optimizer-results/formal/selected_lr_scales.json +Wrote selection report to /root/autodl-tmp/llm-optimizer-results/formal/selected_lr_scales_report.json +Grid-boundary optima require an expanded LR candidate before main: sgd, muon, lion, sf-sgd, signsgd, prodigy, sophiag, adopt, mars, adafactor, lamb, scion, scion-light, d-muon, muon-pytorch +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers adafactor --lr-scales 0.09999999999999999 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[adafactor seed=0 lr_scale=0.1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adafactor --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216 --lr 0.0001 --beta1 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adafactor', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Adafactor ( +Parameter Group 0 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.0001 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.1 + +Parameter Group 1 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.0001 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.464 iter_dt=2.87e-01s lr=1.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.467 iter_dt=2.89e-01s lr=1.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.176 iter_dt=2.90e-01s lr=1.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.141 iter_dt=2.87e-01s lr=1.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=5.998 iter_dt=2.87e-01s lr=1.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.911 val_pp=369.029 val_acc=0.181002 +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers adopt --lr-scales 0.09999999999999999 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[adopt seed=0 lr_scale=0.1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adopt --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216 --lr 0.0001 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adopt', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +ADOPT ( +Parameter Group 0 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.0001 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.0001 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.094 iter_dt=2.72e-01s lr=1.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.878 iter_dt=2.73e-01s lr=1.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.562 iter_dt=2.76e-01s lr=1.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.488 iter_dt=2.72e-01s lr=1.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.294 iter_dt=2.74e-01s lr=1.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.216 val_pp=500.891 val_acc=0.171034 +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers d-muon --lr-scales 9.0 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[d-muon seed=0 lr_scale=9] /root/miniconda3/bin/torchrun --standalone --nproc_per_node=1 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --distributed_backend nccl --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt d-muon --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216 --lr 0.009 --beta1 0.8 --beta2 0.999 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': 'nccl', 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.009, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'd-muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Distributed DataReader Initialized for Worker 0/1 +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +DistributedMuon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.009 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.1 + +Parameter Group 1 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.009 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.713 iter_dt=2.79e-01s lr=9.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=5.791 iter_dt=2.79e-01s lr=9.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=5.484 iter_dt=2.79e-01s lr=9.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.383 iter_dt=2.80e-01s lr=9.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.356 iter_dt=2.81e-01s lr=9.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.192 val_pp=179.777 val_acc=0.210426 +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers lamb --lr-scales 9.0 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[lamb seed=0 lr_scale=9] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt lamb --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216 --lr 0.009 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.009, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lamb', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lamb ( +Parameter Group 0 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.009 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.009 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=9.221 iter_dt=2.79e-01s lr=9.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.767 iter_dt=2.79e-01s lr=9.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.940 iter_dt=2.80e-01s lr=9.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.693 iter_dt=2.84e-01s lr=9.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.400 iter_dt=2.80e-01s lr=9.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.308 val_pp=548.823 val_acc=0.175278 +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers lion --lr-scales 0.09999999999999999 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[lion seed=0 lr_scale=0.1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt lion --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216 --lr 0.0001 --beta1 0.9 --beta2 0.99 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.99, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lion ( +Parameter Group 0 + betas: (0.9, 0.99) + lr: 0.0001 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.99) + lr: 0.0001 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.618 iter_dt=2.74e-01s lr=1.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=7.342 iter_dt=2.83e-01s lr=1.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.871 iter_dt=2.73e-01s lr=1.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.733 iter_dt=2.75e-01s lr=1.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.512 iter_dt=2.74e-01s lr=1.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.423 val_pp=615.991 val_acc=0.147200 +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers mars --lr-scales 0.09999999999999999 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[mars seed=0 lr_scale=0.1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt mars --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216 --lr 0.0001 --mars_lr 0.0003 --beta1 0.8 --beta2 0.999 --mars_beta1 0.95 --mars_beta2 0.99 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'mars', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.0003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +MARS ( +Parameter Group 0 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.0003 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.1 + weight_decay_1d: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.0003 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.0 + weight_decay_1d: 0.1 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.608 iter_dt=2.78e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.717 iter_dt=2.78e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.388 iter_dt=2.78e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.349 iter_dt=2.78e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.189 iter_dt=2.78e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.105 val_pp=448.299 val_acc=0.176981 +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers muon --lr-scales 9.0 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[muon seed=0 lr_scale=9] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt muon --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216 --lr 0.009 --muon_lr_factor 0.09 --beta1 0.8 --beta2 0.999 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.009, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 0.09, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Muon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + adamw_lr: 0.009 + adamw_lr_ratio: 0.09999999999999999 + adamw_wd: 0.1 + lr: 0.09 + momentum: 0.95 + nesterov: True + ns_steps: 5 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.844 iter_dt=2.88e-01s lr=9.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=5.939 iter_dt=2.85e-01s lr=9.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=5.651 iter_dt=2.84e-01s lr=9.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=5.608 iter_dt=2.86e-01s lr=9.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=5.650 iter_dt=2.86e-01s lr=9.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.522 val_pp=250.195 val_acc=0.188259 +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers muon-pytorch --lr-scales 9.0 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[muon-pytorch seed=0 lr_scale=9] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt muon-pytorch --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216 --lr 0.18 --muon_adamw_lr 0.009 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.18, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon-pytorch', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.009, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: + +Train: Iter=50 (0.006104 effective passes) train_loss=6.990 iter_dt=2.76e-01s lr=1.80e-01 +Train: Iter=100 (0.012207 effective passes) train_loss=6.290 iter_dt=2.77e-01s lr=1.80e-01 +Train: Iter=150 (0.018311 effective passes) train_loss=6.096 iter_dt=2.76e-01s lr=1.80e-01 +Train: Iter=200 (0.024414 effective passes) train_loss=6.027 iter_dt=2.77e-01s lr=1.80e-01 +Train: Iter=250 (0.030518 effective passes) train_loss=6.029 iter_dt=2.77e-01s lr=1.80e-01 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.898 val_pp=364.330 val_acc=0.166126 +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers prodigy --lr-scales 0.09999999999999999 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[prodigy seed=0 lr_scale=0.1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt prodigy --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216 --lr 0.1 --beta1 0.9 --beta2 0.999 --prodigy_use_bias_correction True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.1, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'prodigy', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': True, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Using decoupled weight decay + +Optimizer: +Prodigy ( +Parameter Group 0 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 0.1 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.1 + +Parameter Group 1 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 0.1 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=10.893 iter_dt=2.82e-01s lr=1.00e-01 +effective_lr=7.08e-08 +Train: Iter=100 (0.012207 effective passes) train_loss=10.163 iter_dt=2.82e-01s lr=1.00e-01 +effective_lr=3.53e-06 +Train: Iter=150 (0.018311 effective passes) train_loss=7.152 iter_dt=2.82e-01s lr=1.00e-01 +effective_lr=8.62e-05 +Train: Iter=200 (0.024414 effective passes) train_loss=6.522 iter_dt=2.80e-01s lr=1.00e-01 +effective_lr=1.19e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.169 iter_dt=2.82e-01s lr=1.00e-01 +effective_lr=1.54e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.070 val_pp=432.473 val_acc=0.171778 +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers scion --lr-scales 0.09999999999999999 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[scion seed=0 lr_scale=0.1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt scion --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216 --lr 0.0001 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +Scion ( +Parameter Group 0 + lr: 0.0001 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.0001 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.422 iter_dt=2.81e-01s lr=1.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.636 iter_dt=2.82e-01s lr=1.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.472 iter_dt=2.85e-01s lr=1.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.467 iter_dt=2.84e-01s lr=1.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.445 iter_dt=2.84e-01s lr=1.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.358 val_pp=577.189 val_acc=0.143938 +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers scion-light --lr-scales 0.09999999999999999 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[scion-light seed=0 lr_scale=0.1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt scion-light --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216 --lr 0.0001 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion-light', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +ScionLight ( +Parameter Group 0 + lr: 0.0001 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.0001 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.513 iter_dt=2.80e-01s lr=1.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.831 iter_dt=2.89e-01s lr=1.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.682 iter_dt=2.92e-01s lr=1.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.680 iter_dt=2.88e-01s lr=1.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.649 iter_dt=2.88e-01s lr=1.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.559 val_pp=705.505 val_acc=0.138273 +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers sf-sgd --lr-scales 9.0 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[sf-sgd seed=0 lr_scale=9] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sf-sgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216 --lr 0.09 --momentum 0.9 --scheduler none --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.09, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGDScheduleFree ( +Parameter Group 0 + foreach: True + k: 0 + lr: 0.09 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + foreach: True + k: 0 + lr: 0.09 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=9.315 iter_dt=2.72e-01s lr=9.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=8.701 iter_dt=2.70e-01s lr=9.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=8.448 iter_dt=2.72e-01s lr=9.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=8.467 iter_dt=2.72e-01s lr=9.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=8.177 iter_dt=2.82e-01s lr=9.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=8.235 val_pp=3768.939 val_acc=0.068001 +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers sgd --lr-scales 9.0 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[sgd seed=0 lr_scale=9] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216 --lr 0.09 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.09, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGD ( +Parameter Group 0 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.09 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.09 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.301 iter_dt=2.73e-01s lr=9.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=7.720 iter_dt=2.70e-01s lr=9.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=7.597 iter_dt=2.72e-01s lr=9.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=7.730 iter_dt=2.71e-01s lr=9.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=7.597 iter_dt=2.71e-01s lr=9.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.590 val_pp=1978.672 val_acc=0.047283 +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers signsgd --lr-scales 0.09999999999999999 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[signsgd seed=0 lr_scale=0.1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt signsgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216 --lr 0.0001 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'signsgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Signum ( +Parameter Group 0 + dampening: 0.0 + lr: 0.0001 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0.0 + lr: 0.0001 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.674 iter_dt=2.72e-01s lr=1.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=7.313 iter_dt=2.75e-01s lr=1.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.895 iter_dt=2.74e-01s lr=1.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.888 iter_dt=2.70e-01s lr=1.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.720 iter_dt=2.76e-01s lr=1.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.662 val_pp=781.975 val_acc=0.146978 +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers sophiag --lr-scales 0.09999999999999999 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[sophiag seed=0 lr_scale=0.1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sophiag --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216 --lr 0.0001 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sophiag', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SophiaG ( +Parameter Group 0 + betas: (0.9, 0.999) + capturable: False + lr: 0.0001 + maximize: False + rho: 0.04 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + capturable: False + lr: 0.0001 + maximize: False + rho: 0.04 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.622 iter_dt=3.23e-01s lr=1.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=7.197 iter_dt=3.24e-01s lr=1.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.638 iter_dt=3.30e-01s lr=1.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.449 iter_dt=3.23e-01s lr=1.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.202 iter_dt=3.26e-01s lr=1.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.111 val_pp=450.936 val_acc=0.176609 +Wrote selected scales to /root/autodl-tmp/llm-optimizer-results/formal/selected_lr_scales.json +Wrote selection report to /root/autodl-tmp/llm-optimizer-results/formal/selected_lr_scales_report.json +Grid-boundary optima require an expanded LR candidate before main: sgd, sf-sgd, lamb +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers lamb --lr-scales 27.0 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[lamb seed=0 lr_scale=27] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt lamb --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216 --lr 0.027 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.027, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lamb', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lamb ( +Parameter Group 0 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.027 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.027 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.706 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=6.748 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=6.378 iter_dt=2.88e-01s lr=2.70e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=6.262 iter_dt=2.87e-01s lr=2.70e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=6.082 iter_dt=2.84e-01s lr=2.70e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.967 val_pp=390.174 val_acc=0.171759 +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers sf-sgd --lr-scales 27.0 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[sf-sgd seed=0 lr_scale=27] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sf-sgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216 --lr 0.27 --momentum 0.9 --scheduler none --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.27, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGDScheduleFree ( +Parameter Group 0 + foreach: True + k: 0 + lr: 0.27 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + foreach: True + k: 0 + lr: 0.27 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.688 iter_dt=2.72e-01s lr=2.70e-01 +Train: Iter=100 (0.012207 effective passes) train_loss=8.044 iter_dt=2.73e-01s lr=2.70e-01 +Train: Iter=150 (0.018311 effective passes) train_loss=7.837 iter_dt=2.73e-01s lr=2.70e-01 +Train: Iter=200 (0.024414 effective passes) train_loss=7.892 iter_dt=2.72e-01s lr=2.70e-01 +Train: Iter=250 (0.030518 effective passes) train_loss=7.677 iter_dt=2.73e-01s lr=2.70e-01 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.712 val_pp=2234.376 val_acc=0.086718 +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers sgd --lr-scales 27.0 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[sgd seed=0 lr_scale=27] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216 --lr 0.27 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.27, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGD ( +Parameter Group 0 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.27 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.27 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.297 iter_dt=2.72e-01s lr=2.70e-01 +Train: Iter=100 (0.012207 effective passes) train_loss=7.694 iter_dt=2.73e-01s lr=2.70e-01 +Train: Iter=150 (0.018311 effective passes) train_loss=7.655 iter_dt=2.73e-01s lr=2.70e-01 +Train: Iter=200 (0.024414 effective passes) train_loss=7.684 iter_dt=2.76e-01s lr=2.70e-01 +Train: Iter=250 (0.030518 effective passes) train_loss=7.576 iter_dt=2.75e-01s lr=2.70e-01 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.624 val_pp=2046.617 val_acc=0.066933 +Wrote selected scales to /root/autodl-tmp/llm-optimizer-results/formal/selected_lr_scales.json +Wrote selection report to /root/autodl-tmp/llm-optimizer-results/formal/selected_lr_scales_report.json +Grid-boundary optima require an expanded LR candidate before main: sf-sgd, lamb +Traceback (most recent call last): + File "/root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_server_pipeline.py", line 200, in + main() + File "/root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_server_pipeline.py", line 172, in main + selected_path = select_and_expand_lr(args, state_path) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_server_pipeline.py", line 108, in select_and_expand_lr + raise RuntimeError( +RuntimeError: LR optimum remains on a grid boundary after the allowed expansions. Stopping before main. +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[adamw seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adamw --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.8 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[adamw seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adamw --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.8 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[adamw seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adamw --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.8 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[sgd seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216 --lr 0.003 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[sgd seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216 --lr 0.01 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[sgd seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216 --lr 0.03 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[muon seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt muon --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --muon_lr_factor 0.003 --beta1 0.8 --beta2 0.999 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[muon seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt muon --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216 --lr 0.001 --muon_lr_factor 0.01 --beta1 0.8 --beta2 0.999 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[muon seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt muon --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216 --lr 0.003 --muon_lr_factor 0.03 --beta1 0.8 --beta2 0.999 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[soap seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt soap --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[soap seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt soap --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[soap seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt soap --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[ademamix seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt ademamix --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.9 --beta2 0.999 --adema_beta3 0.999 --adema_alpha 8.0 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --adema_beta3_warmup 4096 --adema_alpha_warmup 4096 + +[ademamix seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt ademamix --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.9 --beta2 0.999 --adema_beta3 0.999 --adema_alpha 8.0 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --adema_beta3_warmup 4096 --adema_alpha_warmup 4096 + +[ademamix seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt ademamix --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.9 --beta2 0.999 --adema_beta3 0.999 --adema_alpha 8.0 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --adema_beta3_warmup 4096 --adema_alpha_warmup 4096 + +[lion seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt lion --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.9 --beta2 0.99 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[lion seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt lion --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.9 --beta2 0.99 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[lion seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt lion --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.9 --beta2 0.99 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[sf-adamw seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sf-adamw --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.9 --beta2 0.9999 --scheduler none --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[sf-adamw seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sf-adamw --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.9 --beta2 0.9999 --scheduler none --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[sf-adamw seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sf-adamw --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.9 --beta2 0.9999 --scheduler none --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[sf-sgd seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sf-sgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216 --lr 0.003 --momentum 0.9 --scheduler none --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[sf-sgd seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sf-sgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216 --lr 0.01 --momentum 0.9 --scheduler none --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[sf-sgd seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sf-sgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216 --lr 0.03 --momentum 0.9 --scheduler none --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[signsgd seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt signsgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[signsgd seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt signsgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216 --lr 0.001 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[signsgd seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt signsgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216 --lr 0.003 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[signum seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt signum --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[signum seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt signum --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216 --lr 0.001 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[signum seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt signum --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216 --lr 0.003 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[prodigy seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt prodigy --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216 --lr 0.3 --beta1 0.9 --beta2 0.999 --prodigy_use_bias_correction True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[prodigy seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt prodigy --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216 --lr 1 --beta1 0.9 --beta2 0.999 --prodigy_use_bias_correction True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[prodigy seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt prodigy --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216 --lr 3 --beta1 0.9 --beta2 0.999 --prodigy_use_bias_correction True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[sophiag seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sophiag --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[sophiag seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sophiag --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[sophiag seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sophiag --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[adopt seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adopt --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[adopt seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adopt --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[adopt seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adopt --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[mars seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt mars --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --mars_lr 0.0009 --beta1 0.8 --beta2 0.999 --mars_beta1 0.95 --mars_beta2 0.99 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[mars seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt mars --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216 --lr 0.001 --mars_lr 0.003 --beta1 0.8 --beta2 0.999 --mars_beta1 0.95 --mars_beta2 0.99 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[mars seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt mars --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216 --lr 0.003 --mars_lr 0.009 --beta1 0.8 --beta2 0.999 --mars_beta1 0.95 --mars_beta2 0.99 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[adafactor seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adafactor --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[adafactor seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adafactor --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[adafactor seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adafactor --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[lamb seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt lamb --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[lamb seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt lamb --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[lamb seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt lamb --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[scion seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt scion --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[scion seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt scion --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216 --lr 0.001 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[scion seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt scion --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216 --lr 0.003 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[scion-light seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt scion-light --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[scion-light seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt scion-light --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216 --lr 0.001 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[scion-light seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt scion-light --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216 --lr 0.003 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[d-muon seed=0 lr_scale=0.3] /root/miniconda3/bin/torchrun --standalone --nproc_per_node=1 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --distributed_backend nccl --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt d-muon --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216 --lr 0.0003 --beta1 0.8 --beta2 0.999 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[d-muon seed=0 lr_scale=1] /root/miniconda3/bin/torchrun --standalone --nproc_per_node=1 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --distributed_backend nccl --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt d-muon --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216 --lr 0.001 --beta1 0.8 --beta2 0.999 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[d-muon seed=0 lr_scale=3] /root/miniconda3/bin/torchrun --standalone --nproc_per_node=1 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --distributed_backend nccl --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt d-muon --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216 --lr 0.003 --beta1 0.8 --beta2 0.999 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[muon-pytorch seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt muon-pytorch --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216 --lr 0.006 --muon_adamw_lr 0.0003 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[muon-pytorch seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt muon-pytorch --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216 --lr 0.02 --muon_adamw_lr 0.001 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin + +[muon-pytorch seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt muon-pytorch --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216 --lr 0.06 --muon_adamw_lr 0.003 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Wrote selected scales to /root/autodl-tmp/llm-optimizer-results/formal/selected_lr_scales.json +Wrote selection report to /root/autodl-tmp/llm-optimizer-results/formal/selected_lr_scales_report.json +Grid-boundary optima require an expanded LR candidate before main: sf-sgd, lamb +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers lamb --lr-scales 81.0 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[lamb seed=0 lr_scale=81] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt lamb --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216 --lr 0.081 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.081, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lamb', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lamb ( +Parameter Group 0 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.081 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.081 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.642 iter_dt=2.87e-01s lr=8.10e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=6.959 iter_dt=2.85e-01s lr=8.10e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=6.664 iter_dt=2.82e-01s lr=8.10e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=6.655 iter_dt=2.83e-01s lr=8.10e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=6.605 iter_dt=2.83e-01s lr=8.10e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.549 val_pp=698.800 val_acc=0.122236 +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers sf-sgd --lr-scales 81.0 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[sf-sgd seed=0 lr_scale=81] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sf-sgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216 --lr 0.81 --momentum 0.9 --scheduler none --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.81, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGDScheduleFree ( +Parameter Group 0 + foreach: True + k: 0 + lr: 0.81 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + foreach: True + k: 0 + lr: 0.81 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.361 iter_dt=2.75e-01s lr=8.10e-01 +Train: Iter=100 (0.012207 effective passes) train_loss=7.833 iter_dt=2.73e-01s lr=8.10e-01 +Train: Iter=150 (0.018311 effective passes) train_loss=7.656 iter_dt=2.73e-01s lr=8.10e-01 +Train: Iter=200 (0.024414 effective passes) train_loss=7.798 iter_dt=2.72e-01s lr=8.10e-01 +Train: Iter=250 (0.030518 effective passes) train_loss=7.625 iter_dt=2.75e-01s lr=8.10e-01 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.600 val_pp=1998.699 val_acc=0.081297 +Wrote selected scales to /root/autodl-tmp/llm-optimizer-results/formal/selected_lr_scales.json +Wrote selection report to /root/autodl-tmp/llm-optimizer-results/formal/selected_lr_scales_report.json +Grid-boundary optima require an expanded LR candidate before main: sf-sgd +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers sf-sgd --lr-scales 243.0 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[sf-sgd seed=0 lr_scale=243] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sf-sgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216 --lr 2.43 --momentum 0.9 --scheduler none --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 2.43, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGDScheduleFree ( +Parameter Group 0 + foreach: True + k: 0 + lr: 2.43 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + foreach: True + k: 0 + lr: 2.43 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.163 iter_dt=2.68e-01s lr=2.43e+00 +Train: Iter=100 (0.012207 effective passes) train_loss=7.702 iter_dt=2.67e-01s lr=2.43e+00 +Train: Iter=150 (0.018311 effective passes) train_loss=7.607 iter_dt=2.68e-01s lr=2.43e+00 +Train: Iter=200 (0.024414 effective passes) train_loss=7.747 iter_dt=2.68e-01s lr=2.43e+00 +Train: Iter=250 (0.030518 effective passes) train_loss=7.617 iter_dt=2.72e-01s lr=2.43e+00 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.590 val_pp=1977.611 val_acc=0.052864 +Wrote selected scales to /root/autodl-tmp/llm-optimizer-results/formal/selected_lr_scales.json +Wrote selection report to /root/autodl-tmp/llm-optimizer-results/formal/selected_lr_scales_report.json +Grid-boundary optima require an expanded LR candidate before main: sf-sgd +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage tune --optimizers sf-sgd --lr-scales 729.0 --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[sf-sgd seed=0 lr_scale=729] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sf-sgd --train_token_budget 16777216 --eval_at_tokens 16777216 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 256 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216 --lr 7.29 --momentum 0.9 --scheduler none --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 7.29, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGDScheduleFree ( +Parameter Group 0 + foreach: True + k: 0 + lr: 7.29 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + foreach: True + k: 0 + lr: 7.29 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.415 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=100 (0.012207 effective passes) train_loss=7.823 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=150 (0.018311 effective passes) train_loss=7.725 iter_dt=2.73e-01s lr=7.29e+00 +Train: Iter=200 (0.024414 effective passes) train_loss=7.884 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=250 (0.030518 effective passes) train_loss=7.629 iter_dt=2.73e-01s lr=7.29e+00 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.577 val_pp=1953.157 val_acc=0.050619 +Wrote selected scales to /root/autodl-tmp/llm-optimizer-results/formal/selected_lr_scales.json +Wrote selection report to /root/autodl-tmp/llm-optimizer-results/formal/selected_lr_scales_report.json +Grid-boundary optima require an expanded LR candidate before main: sf-sgd +Traceback (most recent call last): + File "/root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_server_pipeline.py", line 200, in + main() + File "/root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_server_pipeline.py", line 172, in main + selected_path = select_and_expand_lr(args, state_path) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_server_pipeline.py", line 108, in select_and_expand_lr + raise RuntimeError( +RuntimeError: LR optimum remains on a grid boundary after the allowed expansions. Stopping before main. +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/run_all.py --stage main --lr-scale-file /root/autodl-tmp/llm-optimizer-results/formal/selected_lr_scales.json --dataset token-bin --train-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val-data-path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --results-dir /root/autodl-tmp/llm-optimizer-results/formal + +[adamw seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adamw --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456 --lr 0.001 --beta1 0.8 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +using fused AdamW: True + +Optimizer: +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.001 + maximize: False + weight_decay: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.001 + maximize: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.310 iter_dt=2.66e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.401 iter_dt=2.66e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.041 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.931 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.765 iter_dt=2.69e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.626 val_pp=277.625 val_acc=0.185545 +Train: Iter=300 (0.036622 effective passes) train_loss=5.397 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.556 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=5.272 iter_dt=2.66e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=5.019 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=4.976 iter_dt=2.67e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.043 val_pp=154.935 val_acc=0.224113 +Train: Iter=550 (0.067140 effective passes) train_loss=5.079 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.652 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=5.013 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.886 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.867 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.665 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.612 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.558 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.602 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.635 iter_dt=2.69e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.545 val_pp=94.133 val_acc=0.271013 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.642 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.455 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.481 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.411 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.429 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.467 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.493 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.484 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.345 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.425 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.356 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.624 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.200 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.546 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.203 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.254 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.545 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.148 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.107 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.295 iter_dt=2.68e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.271 val_pp=71.603 val_acc=0.293463 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.286 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.291 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.259 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.247 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.298 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.990 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.190 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.180 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.175 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.438 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.216 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.993 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.446 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.310 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.214 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.047 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.354 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.265 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.216 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.058 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.225 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.859 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.093 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.177 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.237 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.109 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.313 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.032 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.005 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.093 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.165 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.255 iter_dt=2.64e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.177 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.061 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.064 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.158 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.985 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.094 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.142 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.200 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.077 iter_dt=2.69e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.083 val_pp=59.349 val_acc=0.308804 + +[sgd seed=0 lr_scale=9] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sgd --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456 --lr 0.09 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.09, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGD ( +Parameter Group 0 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.09 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.09 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.292 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=7.689 iter_dt=2.69e-01s lr=9.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=7.567 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=7.721 iter_dt=2.69e-01s lr=9.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=7.588 iter_dt=2.69e-01s lr=9.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.568 val_pp=1935.208 val_acc=0.051807 +Train: Iter=300 (0.036622 effective passes) train_loss=7.692 iter_dt=2.70e-01s lr=9.00e-02 +Train: Iter=350 (0.042725 effective passes) train_loss=7.563 iter_dt=2.70e-01s lr=9.00e-02 +Train: Iter=400 (0.048829 effective passes) train_loss=7.522 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=450 (0.054932 effective passes) train_loss=7.464 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=500 (0.061036 effective passes) train_loss=7.381 iter_dt=2.68e-01s lr=9.00e-02 +>Eval: Iter=512 (0.062501 effective passes) val_loss=7.446 val_pp=1713.474 val_acc=0.052227 +Train: Iter=550 (0.067140 effective passes) train_loss=7.624 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=600 (0.073243 effective passes) train_loss=7.130 iter_dt=2.74e-01s lr=9.00e-02 +Train: Iter=650 (0.079347 effective passes) train_loss=7.596 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=700 (0.085451 effective passes) train_loss=7.345 iter_dt=2.74e-01s lr=9.00e-02 +Train: Iter=750 (0.091554 effective passes) train_loss=7.326 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=800 (0.097658 effective passes) train_loss=7.391 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=850 (0.103761 effective passes) train_loss=7.338 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=900 (0.109865 effective passes) train_loss=7.294 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=950 (0.115969 effective passes) train_loss=7.228 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=1000 (0.122072 effective passes) train_loss=7.240 iter_dt=2.68e-01s lr=9.00e-02 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=7.362 val_pp=1574.316 val_acc=0.049957 +Train: Iter=1050 (0.128176 effective passes) train_loss=7.425 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=1100 (0.134279 effective passes) train_loss=7.381 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=1150 (0.140383 effective passes) train_loss=7.204 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=1200 (0.146487 effective passes) train_loss=7.232 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=1250 (0.152590 effective passes) train_loss=7.276 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=1300 (0.158694 effective passes) train_loss=7.364 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=1350 (0.164797 effective passes) train_loss=7.217 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=1400 (0.170901 effective passes) train_loss=7.280 iter_dt=2.69e-01s lr=9.00e-02 +Train: Iter=1450 (0.177005 effective passes) train_loss=7.224 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=1500 (0.183108 effective passes) train_loss=7.324 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=1550 (0.189212 effective passes) train_loss=7.164 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=1600 (0.195315 effective passes) train_loss=7.403 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=1650 (0.201419 effective passes) train_loss=7.120 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=1700 (0.207523 effective passes) train_loss=7.480 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=1750 (0.213626 effective passes) train_loss=7.257 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=1800 (0.219730 effective passes) train_loss=7.180 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=1850 (0.225834 effective passes) train_loss=7.318 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=1900 (0.231937 effective passes) train_loss=7.179 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=1950 (0.238041 effective passes) train_loss=7.385 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=2000 (0.244144 effective passes) train_loss=7.359 iter_dt=2.66e-01s lr=9.00e-02 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=7.295 val_pp=1473.470 val_acc=0.057417 +Train: Iter=2050 (0.250248 effective passes) train_loss=7.426 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=2100 (0.256352 effective passes) train_loss=7.263 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=2150 (0.262455 effective passes) train_loss=7.308 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=2200 (0.268559 effective passes) train_loss=7.084 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=2250 (0.274662 effective passes) train_loss=7.161 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=2300 (0.280766 effective passes) train_loss=7.262 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=2350 (0.286870 effective passes) train_loss=7.032 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=2400 (0.292973 effective passes) train_loss=7.418 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=2450 (0.299077 effective passes) train_loss=6.993 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=2500 (0.305180 effective passes) train_loss=7.202 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=2550 (0.311284 effective passes) train_loss=7.406 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=2600 (0.317388 effective passes) train_loss=6.963 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=2650 (0.323491 effective passes) train_loss=7.071 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=2700 (0.329595 effective passes) train_loss=7.203 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=2750 (0.335698 effective passes) train_loss=7.121 iter_dt=2.61e-01s lr=9.00e-02 +Train: Iter=2800 (0.341802 effective passes) train_loss=7.113 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=2850 (0.347906 effective passes) train_loss=7.234 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=2900 (0.354009 effective passes) train_loss=7.172 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=2950 (0.360113 effective passes) train_loss=7.074 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=3000 (0.366217 effective passes) train_loss=6.886 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3050 (0.372320 effective passes) train_loss=7.227 iter_dt=2.69e-01s lr=9.00e-02 +Train: Iter=3100 (0.378424 effective passes) train_loss=6.599 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=3150 (0.384527 effective passes) train_loss=6.945 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=3200 (0.390631 effective passes) train_loss=6.938 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=3250 (0.396735 effective passes) train_loss=7.133 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=3300 (0.402838 effective passes) train_loss=7.157 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3350 (0.408942 effective passes) train_loss=7.027 iter_dt=2.69e-01s lr=9.00e-02 +Train: Iter=3400 (0.415045 effective passes) train_loss=6.879 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=3450 (0.421149 effective passes) train_loss=6.841 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=3500 (0.427253 effective passes) train_loss=6.870 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3550 (0.433356 effective passes) train_loss=7.156 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3600 (0.439460 effective passes) train_loss=7.161 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3650 (0.445563 effective passes) train_loss=7.119 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3700 (0.451667 effective passes) train_loss=6.961 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3750 (0.457771 effective passes) train_loss=7.021 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3800 (0.463874 effective passes) train_loss=7.404 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=3850 (0.469978 effective passes) train_loss=6.866 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3900 (0.476081 effective passes) train_loss=6.835 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3950 (0.482185 effective passes) train_loss=6.974 iter_dt=2.65e-01s lr=9.00e-02 +Train: Iter=4000 (0.488289 effective passes) train_loss=7.325 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=4050 (0.494392 effective passes) train_loss=6.804 iter_dt=2.66e-01s lr=9.00e-02 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=6.973 val_pp=1067.516 val_acc=0.103691 + +[muon seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt muon --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456 --lr 0.003 --muon_lr_factor 0.03 --beta1 0.8 --beta2 0.999 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 0.03, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Muon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + adamw_lr: 0.003 + adamw_lr_ratio: 0.1 + adamw_wd: 0.1 + lr: 0.03 + momentum: 0.95 + nesterov: True + ns_steps: 5 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.662 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=5.824 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=5.501 iter_dt=2.91e-01s lr=3.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=5.363 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=5.333 iter_dt=2.84e-01s lr=3.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.168 val_pp=175.532 val_acc=0.212906 +Train: Iter=300 (0.036622 effective passes) train_loss=5.054 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=350 (0.042725 effective passes) train_loss=5.160 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=400 (0.048829 effective passes) train_loss=4.787 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=450 (0.054932 effective passes) train_loss=4.599 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=500 (0.061036 effective passes) train_loss=4.627 iter_dt=2.86e-01s lr=3.00e-02 +>Eval: Iter=512 (0.062501 effective passes) val_loss=4.705 val_pp=110.539 val_acc=0.255857 +Train: Iter=550 (0.067140 effective passes) train_loss=4.684 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=600 (0.073243 effective passes) train_loss=4.426 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=650 (0.079347 effective passes) train_loss=4.802 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=700 (0.085451 effective passes) train_loss=4.681 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=750 (0.091554 effective passes) train_loss=4.691 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=800 (0.097658 effective passes) train_loss=4.515 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=850 (0.103761 effective passes) train_loss=4.455 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=900 (0.109865 effective passes) train_loss=4.394 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=950 (0.115969 effective passes) train_loss=4.481 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.525 iter_dt=2.86e-01s lr=3.00e-02 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.409 val_pp=82.167 val_acc=0.280989 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.533 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.338 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.364 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.300 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.308 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.353 iter_dt=3.01e-01s lr=3.00e-02 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.383 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.394 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.261 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.316 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.270 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.550 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.124 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.460 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.138 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.167 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.469 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.072 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.011 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.200 iter_dt=2.87e-01s lr=3.00e-02 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.191 val_pp=66.120 val_acc=0.300385 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.208 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.221 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.193 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.182 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.213 iter_dt=2.97e-01s lr=3.00e-02 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.917 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.121 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.110 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.115 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.366 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.156 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.949 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.366 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.224 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.137 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.997 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.290 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.198 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.154 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.003 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.165 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.809 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.033 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.111 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.170 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.047 iter_dt=2.92e-01s lr=3.00e-02 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.267 iter_dt=2.91e-01s lr=3.00e-02 +Train: Iter=3400 (0.415045 effective passes) train_loss=3.981 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.951 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.039 iter_dt=2.92e-01s lr=3.00e-02 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.105 iter_dt=3.00e-01s lr=3.00e-02 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.208 iter_dt=2.94e-01s lr=3.00e-02 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.151 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.030 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.017 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.095 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.938 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.044 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.118 iter_dt=2.92e-01s lr=3.00e-02 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.133 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.044 iter_dt=2.90e-01s lr=3.00e-02 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.044 val_pp=57.034 val_acc=0.313597 + +[soap seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt soap --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456 --lr 0.001 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'soap', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SOAP ( +Parameter Group 0 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.001 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.001 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.365 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.134 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=5.701 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.556 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.445 iter_dt=2.84e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.292 val_pp=198.774 val_acc=0.209633 +Train: Iter=300 (0.036622 effective passes) train_loss=5.092 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.259 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=4.864 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=4.606 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=4.637 iter_dt=2.81e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=4.701 val_pp=110.016 val_acc=0.257669 +Train: Iter=550 (0.067140 effective passes) train_loss=4.668 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.415 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=4.755 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.643 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.660 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.478 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.418 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.371 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.452 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.494 iter_dt=2.82e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.373 val_pp=79.320 val_acc=0.284046 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.506 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.318 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.338 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.262 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.295 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.338 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.358 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.374 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.241 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.309 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.255 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.525 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.103 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.433 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.118 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.144 iter_dt=2.87e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.451 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.053 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=3.993 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.224 iter_dt=2.82e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.174 val_pp=64.982 val_acc=0.300755 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.174 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.193 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.164 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.149 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.203 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.913 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.112 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.101 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.123 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.383 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.144 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.945 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.371 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.230 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.131 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.983 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.293 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.194 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.150 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=3.986 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.172 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.790 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.022 iter_dt=2.86e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.104 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.172 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.030 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.253 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=3.979 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.940 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.057 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.107 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.182 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.122 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.013 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=3.993 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.094 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.930 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.049 iter_dt=2.86e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.102 iter_dt=2.89e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.133 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.037 iter_dt=2.83e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.039 val_pp=56.759 val_acc=0.311636 + +[ademamix seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt ademamix --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456 --lr 0.001 --beta1 0.9 --beta2 0.999 --adema_beta3 0.999 --adema_alpha 8.0 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin --adema_beta3_warmup 4096 --adema_alpha_warmup 4096 +Starting Experiment: sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'ademamix', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.999, 'adema_alpha': 8.0, 'adema_beta3_warmup': 4096, 'adema_alpha_warmup': 4096, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdEMAMix ( +Parameter Group 0 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.001 + weight_decay: 0.1 + +Parameter Group 1 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.001 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.303 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.491 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.144 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.979 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.822 iter_dt=2.76e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.677 val_pp=292.054 val_acc=0.181980 +Train: Iter=300 (0.036622 effective passes) train_loss=5.413 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.595 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=5.294 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=5.083 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=4.992 iter_dt=2.78e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.065 val_pp=158.348 val_acc=0.220623 +Train: Iter=550 (0.067140 effective passes) train_loss=5.108 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.668 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=5.023 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.867 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.852 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.660 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.561 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.506 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.572 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.591 iter_dt=2.75e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.498 val_pp=89.869 val_acc=0.274141 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.617 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.431 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.441 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.365 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.383 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.432 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.425 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.452 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.298 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.382 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.292 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.572 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.142 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.495 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.165 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.181 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.489 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.077 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.032 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.236 iter_dt=2.74e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.206 val_pp=67.078 val_acc=0.299908 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.220 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.234 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.190 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.176 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.235 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.932 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.118 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.110 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.117 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.385 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.161 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.943 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.368 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.238 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.117 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.996 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.293 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.201 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.152 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=3.995 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.171 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.800 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.038 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.115 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.158 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.026 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.240 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=3.966 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.951 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.018 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.101 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.172 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.111 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.009 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=3.984 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.080 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.916 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.021 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.096 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.108 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.019 iter_dt=2.74e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.020 val_pp=55.698 val_acc=0.316416 + +[lion seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt lion --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456 --lr 0.0003 --beta1 0.9 --beta2 0.99 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.99, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lion ( +Parameter Group 0 + betas: (0.9, 0.99) + lr: 0.0003 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.99) + lr: 0.0003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.616 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.890 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.661 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.575 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.403 iter_dt=2.73e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.305 val_pp=547.424 val_acc=0.142797 +Train: Iter=300 (0.036622 effective passes) train_loss=6.054 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=350 (0.042725 effective passes) train_loss=6.198 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=400 (0.048829 effective passes) train_loss=5.938 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=450 (0.054932 effective passes) train_loss=5.725 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=500 (0.061036 effective passes) train_loss=5.603 iter_dt=2.75e-01s lr=3.00e-04 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.674 val_pp=291.197 val_acc=0.180149 +Train: Iter=550 (0.067140 effective passes) train_loss=5.809 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=600 (0.073243 effective passes) train_loss=5.239 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=650 (0.079347 effective passes) train_loss=5.693 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=700 (0.085451 effective passes) train_loss=5.442 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=750 (0.091554 effective passes) train_loss=5.414 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=800 (0.097658 effective passes) train_loss=5.194 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=850 (0.103761 effective passes) train_loss=5.174 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=900 (0.109865 effective passes) train_loss=5.087 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=950 (0.115969 effective passes) train_loss=5.102 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=1000 (0.122072 effective passes) train_loss=5.069 iter_dt=2.72e-01s lr=3.00e-04 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=5.056 val_pp=156.948 val_acc=0.222181 +Train: Iter=1050 (0.128176 effective passes) train_loss=5.153 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.953 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.881 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.850 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.825 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.920 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.827 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.820 iter_dt=2.76e-01s lr=3.00e-04 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.693 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.757 iter_dt=2.76e-01s lr=3.00e-04 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.634 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.917 iter_dt=2.79e-01s lr=3.00e-04 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.470 iter_dt=2.77e-01s lr=3.00e-04 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.802 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.485 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.494 iter_dt=2.76e-01s lr=3.00e-04 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.769 iter_dt=2.76e-01s lr=3.00e-04 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.373 iter_dt=2.76e-01s lr=3.00e-04 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.346 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.516 iter_dt=2.75e-01s lr=3.00e-04 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.471 val_pp=87.436 val_acc=0.277426 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.485 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.489 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.436 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.408 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.445 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.174 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.333 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.367 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.345 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.595 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.372 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.124 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.555 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.435 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.323 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.166 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.471 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.407 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.317 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.164 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.344 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.959 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.222 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.278 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.347 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.213 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.414 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.139 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.085 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.179 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.261 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.338 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.282 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.128 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.142 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.255 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.052 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.155 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.216 iter_dt=2.78e-01s lr=3.00e-04 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.272 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.159 iter_dt=2.72e-01s lr=3.00e-04 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.167 val_pp=64.520 val_acc=0.303646 + +[sf-adamw seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sf-adamw --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456 --lr 0.001 --beta1 0.9 --beta2 0.9999 --scheduler none --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.9999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdamWScheduleFree ( +Parameter Group 0 + betas: (0.9, 0.9999) + eps: 1e-08 + foreach: True + k: 0 + lr: 0.001 + lr_max: -1.0 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + betas: (0.9, 0.9999) + eps: 1e-08 + foreach: True + k: 0 + lr: 0.001 + lr_max: -1.0 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.243 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.466 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.221 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.203 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.082 iter_dt=2.72e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.023 val_pp=412.680 val_acc=0.167620 +Train: Iter=300 (0.036622 effective passes) train_loss=5.870 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.973 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=5.712 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=5.518 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=5.452 iter_dt=2.73e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.568 val_pp=261.798 val_acc=0.191311 +Train: Iter=550 (0.067140 effective passes) train_loss=5.664 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=5.115 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=5.622 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=5.377 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=5.374 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=5.161 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=5.175 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=5.111 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=5.121 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=5.078 iter_dt=2.74e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=5.125 val_pp=168.201 val_acc=0.217758 +Train: Iter=1050 (0.128176 effective passes) train_loss=5.206 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=5.012 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.948 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.926 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.909 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=5.004 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.944 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.899 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.820 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.885 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.778 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=5.100 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.659 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.984 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.678 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.666 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.942 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.594 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.561 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.734 iter_dt=2.72e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.691 val_pp=108.955 val_acc=0.251253 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.705 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.686 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.641 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.546 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.641 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.401 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.491 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.581 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.497 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.752 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.573 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.283 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.679 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.593 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.520 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.357 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.640 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.575 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.453 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.349 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.509 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=4.095 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.360 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.415 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.480 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.385 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.569 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.244 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.227 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.312 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.408 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.471 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.419 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.257 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.286 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.403 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.201 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.292 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.358 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.403 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.268 iter_dt=2.72e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.305 val_pp=74.039 val_acc=0.290781 + +[sf-sgd seed=0 lr_scale=729] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sf-sgd --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456 --lr 7.29 --momentum 0.9 --scheduler none --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 7.29, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGDScheduleFree ( +Parameter Group 0 + foreach: True + k: 0 + lr: 7.29 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + foreach: True + k: 0 + lr: 7.29 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.460 iter_dt=2.73e-01s lr=7.29e+00 +Train: Iter=100 (0.012207 effective passes) train_loss=7.914 iter_dt=2.74e-01s lr=7.29e+00 +Train: Iter=150 (0.018311 effective passes) train_loss=7.844 iter_dt=2.72e-01s lr=7.29e+00 +Train: Iter=200 (0.024414 effective passes) train_loss=7.829 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=250 (0.030518 effective passes) train_loss=8.025 iter_dt=2.75e-01s lr=7.29e+00 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.614 val_pp=2027.040 val_acc=0.053434 +Train: Iter=300 (0.036622 effective passes) train_loss=7.872 iter_dt=2.73e-01s lr=7.29e+00 +Train: Iter=350 (0.042725 effective passes) train_loss=7.702 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=400 (0.048829 effective passes) train_loss=7.595 iter_dt=2.72e-01s lr=7.29e+00 +Train: Iter=450 (0.054932 effective passes) train_loss=7.604 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=500 (0.061036 effective passes) train_loss=8.115 iter_dt=2.70e-01s lr=7.29e+00 +>Eval: Iter=512 (0.062501 effective passes) val_loss=7.474 val_pp=1760.878 val_acc=0.059734 +Train: Iter=550 (0.067140 effective passes) train_loss=7.745 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=600 (0.073243 effective passes) train_loss=7.165 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=650 (0.079347 effective passes) train_loss=7.732 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=700 (0.085451 effective passes) train_loss=7.511 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=750 (0.091554 effective passes) train_loss=7.480 iter_dt=2.72e-01s lr=7.29e+00 +Train: Iter=800 (0.097658 effective passes) train_loss=8.234 iter_dt=2.79e-01s lr=7.29e+00 +Train: Iter=850 (0.103761 effective passes) train_loss=7.467 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=900 (0.109865 effective passes) train_loss=7.445 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=950 (0.115969 effective passes) train_loss=8.328 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=1000 (0.122072 effective passes) train_loss=7.271 iter_dt=2.71e-01s lr=7.29e+00 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=7.332 val_pp=1529.089 val_acc=0.084913 +Train: Iter=1050 (0.128176 effective passes) train_loss=7.482 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=1100 (0.134279 effective passes) train_loss=7.431 iter_dt=2.76e-01s lr=7.29e+00 +Train: Iter=1150 (0.140383 effective passes) train_loss=7.354 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=1200 (0.146487 effective passes) train_loss=7.219 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=1250 (0.152590 effective passes) train_loss=7.848 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=1300 (0.158694 effective passes) train_loss=7.812 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=1350 (0.164797 effective passes) train_loss=7.574 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=1400 (0.170901 effective passes) train_loss=7.360 iter_dt=2.72e-01s lr=7.29e+00 +Train: Iter=1450 (0.177005 effective passes) train_loss=7.215 iter_dt=2.69e-01s lr=7.29e+00 +Train: Iter=1500 (0.183108 effective passes) train_loss=8.048 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=1550 (0.189212 effective passes) train_loss=10.113 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=1600 (0.195315 effective passes) train_loss=7.383 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=1650 (0.201419 effective passes) train_loss=7.066 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=1700 (0.207523 effective passes) train_loss=7.618 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=1750 (0.213626 effective passes) train_loss=7.235 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=1800 (0.219730 effective passes) train_loss=7.287 iter_dt=2.73e-01s lr=7.29e+00 +Train: Iter=1850 (0.225834 effective passes) train_loss=7.231 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=1900 (0.231937 effective passes) train_loss=7.089 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=1950 (0.238041 effective passes) train_loss=7.491 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=2000 (0.244144 effective passes) train_loss=7.319 iter_dt=2.69e-01s lr=7.29e+00 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=7.188 val_pp=1323.053 val_acc=0.089077 +Train: Iter=2050 (0.250248 effective passes) train_loss=7.757 iter_dt=2.69e-01s lr=7.29e+00 +Train: Iter=2100 (0.256352 effective passes) train_loss=7.333 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=2150 (0.262455 effective passes) train_loss=7.246 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=2200 (0.268559 effective passes) train_loss=7.126 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=2250 (0.274662 effective passes) train_loss=7.118 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=2300 (0.280766 effective passes) train_loss=7.445 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=2350 (0.286870 effective passes) train_loss=7.118 iter_dt=2.69e-01s lr=7.29e+00 +Train: Iter=2400 (0.292973 effective passes) train_loss=7.360 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=2450 (0.299077 effective passes) train_loss=7.061 iter_dt=2.69e-01s lr=7.29e+00 +Train: Iter=2500 (0.305180 effective passes) train_loss=7.238 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=2550 (0.311284 effective passes) train_loss=7.577 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=2600 (0.317388 effective passes) train_loss=7.003 iter_dt=2.69e-01s lr=7.29e+00 +Train: Iter=2650 (0.323491 effective passes) train_loss=7.327 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=2700 (0.329595 effective passes) train_loss=7.497 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=2750 (0.335698 effective passes) train_loss=7.135 iter_dt=2.69e-01s lr=7.29e+00 +Train: Iter=2800 (0.341802 effective passes) train_loss=7.167 iter_dt=2.69e-01s lr=7.29e+00 +Train: Iter=2850 (0.347906 effective passes) train_loss=7.513 iter_dt=2.69e-01s lr=7.29e+00 +Train: Iter=2900 (0.354009 effective passes) train_loss=7.237 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=2950 (0.360113 effective passes) train_loss=7.276 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3000 (0.366217 effective passes) train_loss=7.055 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=3050 (0.372320 effective passes) train_loss=7.489 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3100 (0.378424 effective passes) train_loss=7.820 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=3150 (0.384527 effective passes) train_loss=7.126 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3200 (0.390631 effective passes) train_loss=7.126 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3250 (0.396735 effective passes) train_loss=7.130 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3300 (0.402838 effective passes) train_loss=7.407 iter_dt=2.87e-01s lr=7.29e+00 +Train: Iter=3350 (0.408942 effective passes) train_loss=7.120 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3400 (0.415045 effective passes) train_loss=7.722 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3450 (0.421149 effective passes) train_loss=8.052 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3500 (0.427253 effective passes) train_loss=7.158 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3550 (0.433356 effective passes) train_loss=7.856 iter_dt=2.74e-01s lr=7.29e+00 +Train: Iter=3600 (0.439460 effective passes) train_loss=7.351 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=3650 (0.445563 effective passes) train_loss=7.519 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=3700 (0.451667 effective passes) train_loss=7.244 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=3750 (0.457771 effective passes) train_loss=7.126 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=3800 (0.463874 effective passes) train_loss=7.649 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3850 (0.469978 effective passes) train_loss=7.070 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=3900 (0.476081 effective passes) train_loss=7.072 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=3950 (0.482185 effective passes) train_loss=7.074 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=4000 (0.488289 effective passes) train_loss=7.296 iter_dt=2.69e-01s lr=7.29e+00 +Train: Iter=4050 (0.494392 effective passes) train_loss=6.844 iter_dt=2.69e-01s lr=7.29e+00 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=6.992 val_pp=1088.043 val_acc=0.106506 + +[signsgd seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt signsgd --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456 --lr 0.0003 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'signsgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Signum ( +Parameter Group 0 + dampening: 0.0 + lr: 0.0003 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0.0 + lr: 0.0003 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.539 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.728 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.583 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.670 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.571 iter_dt=2.71e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.492 val_pp=659.833 val_acc=0.152500 +Train: Iter=300 (0.036622 effective passes) train_loss=6.601 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=350 (0.042725 effective passes) train_loss=6.515 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=400 (0.048829 effective passes) train_loss=6.368 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=450 (0.054932 effective passes) train_loss=6.171 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=500 (0.061036 effective passes) train_loss=6.097 iter_dt=2.71e-01s lr=3.00e-04 +>Eval: Iter=512 (0.062501 effective passes) val_loss=6.202 val_pp=493.816 val_acc=0.169327 +Train: Iter=550 (0.067140 effective passes) train_loss=6.396 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=600 (0.073243 effective passes) train_loss=5.762 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=650 (0.079347 effective passes) train_loss=6.318 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=700 (0.085451 effective passes) train_loss=6.025 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=750 (0.091554 effective passes) train_loss=6.045 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=800 (0.097658 effective passes) train_loss=5.925 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=850 (0.103761 effective passes) train_loss=5.900 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=900 (0.109865 effective passes) train_loss=5.872 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=950 (0.115969 effective passes) train_loss=5.782 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=1000 (0.122072 effective passes) train_loss=5.772 iter_dt=2.70e-01s lr=3.00e-04 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=5.865 val_pp=352.379 val_acc=0.191597 +Train: Iter=1050 (0.128176 effective passes) train_loss=5.953 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=1100 (0.134279 effective passes) train_loss=5.856 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=1150 (0.140383 effective passes) train_loss=5.686 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=1200 (0.146487 effective passes) train_loss=5.683 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=1250 (0.152590 effective passes) train_loss=5.705 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=1300 (0.158694 effective passes) train_loss=5.776 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1350 (0.164797 effective passes) train_loss=5.659 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=1400 (0.170901 effective passes) train_loss=5.649 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=1450 (0.177005 effective passes) train_loss=5.544 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=1500 (0.183108 effective passes) train_loss=5.745 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=1550 (0.189212 effective passes) train_loss=5.534 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=1600 (0.195315 effective passes) train_loss=5.869 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=1650 (0.201419 effective passes) train_loss=5.406 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=1700 (0.207523 effective passes) train_loss=5.875 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=1750 (0.213626 effective passes) train_loss=5.541 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1800 (0.219730 effective passes) train_loss=5.507 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=1850 (0.225834 effective passes) train_loss=5.714 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=1900 (0.231937 effective passes) train_loss=5.422 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=1950 (0.238041 effective passes) train_loss=5.586 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2000 (0.244144 effective passes) train_loss=5.616 iter_dt=2.71e-01s lr=3.00e-04 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=5.576 val_pp=263.959 val_acc=0.212559 +Train: Iter=2050 (0.250248 effective passes) train_loss=5.729 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=2100 (0.256352 effective passes) train_loss=5.579 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=2150 (0.262455 effective passes) train_loss=5.583 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=2200 (0.268559 effective passes) train_loss=5.370 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=2250 (0.274662 effective passes) train_loss=5.488 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=2300 (0.280766 effective passes) train_loss=5.476 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=2350 (0.286870 effective passes) train_loss=5.334 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=2400 (0.292973 effective passes) train_loss=5.752 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=2450 (0.299077 effective passes) train_loss=5.321 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=2500 (0.305180 effective passes) train_loss=5.643 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=2550 (0.311284 effective passes) train_loss=5.698 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=2600 (0.317388 effective passes) train_loss=5.170 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=2650 (0.323491 effective passes) train_loss=5.424 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=2700 (0.329595 effective passes) train_loss=5.584 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2750 (0.335698 effective passes) train_loss=5.541 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2800 (0.341802 effective passes) train_loss=5.367 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2850 (0.347906 effective passes) train_loss=5.577 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2900 (0.354009 effective passes) train_loss=5.577 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2950 (0.360113 effective passes) train_loss=5.375 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3000 (0.366217 effective passes) train_loss=5.248 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3050 (0.372320 effective passes) train_loss=5.492 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3100 (0.378424 effective passes) train_loss=4.825 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3150 (0.384527 effective passes) train_loss=5.288 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3200 (0.390631 effective passes) train_loss=5.358 iter_dt=2.91e-01s lr=3.00e-04 +Train: Iter=3250 (0.396735 effective passes) train_loss=5.480 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3300 (0.402838 effective passes) train_loss=5.455 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3350 (0.408942 effective passes) train_loss=5.400 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3400 (0.415045 effective passes) train_loss=5.111 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3450 (0.421149 effective passes) train_loss=5.131 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3500 (0.427253 effective passes) train_loss=5.249 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3550 (0.433356 effective passes) train_loss=5.417 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3600 (0.439460 effective passes) train_loss=5.539 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3650 (0.445563 effective passes) train_loss=5.457 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3700 (0.451667 effective passes) train_loss=5.238 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3750 (0.457771 effective passes) train_loss=5.355 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3800 (0.463874 effective passes) train_loss=5.727 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3850 (0.469978 effective passes) train_loss=5.156 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3900 (0.476081 effective passes) train_loss=5.165 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3950 (0.482185 effective passes) train_loss=5.321 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=4000 (0.488289 effective passes) train_loss=5.578 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=4050 (0.494392 effective passes) train_loss=5.169 iter_dt=2.74e-01s lr=3.00e-04 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=5.292 val_pp=198.721 val_acc=0.240356 + +[signum seed=0 lr_scale=1] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt signum --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456 --lr 0.001 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'signum', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Signum ( +Parameter Group 0 + dampening: 0.0 + lr: 0.001 + momentum: 0.9 + nesterov: False + sign_update: True + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0.0 + lr: 0.001 + momentum: 0.9 + nesterov: False + sign_update: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.888 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.360 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.706 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.528 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.291 iter_dt=2.70e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.178 val_pp=481.885 val_acc=0.151791 +Train: Iter=300 (0.036622 effective passes) train_loss=5.848 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=6.034 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=5.696 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=5.514 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=5.361 iter_dt=2.73e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.435 val_pp=229.296 val_acc=0.195808 +Train: Iter=550 (0.067140 effective passes) train_loss=5.522 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.996 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=5.424 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=5.181 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=5.153 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.975 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.886 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.811 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.867 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.858 iter_dt=2.76e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.796 val_pp=121.082 val_acc=0.251291 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.890 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.709 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.704 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.625 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.643 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.693 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.681 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.686 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.541 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.655 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.543 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.839 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.401 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.746 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.412 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.430 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.736 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.339 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.308 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.489 iter_dt=2.75e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.458 val_pp=86.295 val_acc=0.278936 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.503 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.465 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.439 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.424 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.454 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.191 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.361 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.377 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.354 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.634 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.406 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.171 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.571 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.464 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.375 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.219 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.516 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.446 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.371 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.215 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.382 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.993 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.254 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.333 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.408 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.283 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.448 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.179 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.162 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.267 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.330 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.422 iter_dt=2.65e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.344 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.222 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.223 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.333 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.136 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.243 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.282 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.352 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.229 iter_dt=2.72e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.240 val_pp=69.406 val_acc=0.297480 + +[prodigy seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt prodigy --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456 --lr 0.3 --beta1 0.9 --beta2 0.999 --prodigy_use_bias_correction True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.3, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'prodigy', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': True, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Using decoupled weight decay + +Optimizer: +Prodigy ( +Parameter Group 0 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 0.3 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.1 + +Parameter Group 1 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 0.3 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=10.238 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=6.14e-06 +Train: Iter=100 (0.012207 effective passes) train_loss=6.907 iter_dt=2.82e-01s lr=3.00e-01 +effective_lr=1.50e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.316 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=1.82e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.168 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=2.95e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=5.929 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=3.78e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.788 val_pp=326.202 val_acc=0.179335 +Train: Iter=300 (0.036622 effective passes) train_loss=5.558 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=4.43e-04 +Train: Iter=350 (0.042725 effective passes) train_loss=5.682 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=4.99e-04 +Train: Iter=400 (0.048829 effective passes) train_loss=5.337 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=5.47e-04 +Train: Iter=450 (0.054932 effective passes) train_loss=5.105 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=5.76e-04 +Train: Iter=500 (0.061036 effective passes) train_loss=5.051 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=6.07e-04 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.106 val_pp=164.941 val_acc=0.218599 +Train: Iter=550 (0.067140 effective passes) train_loss=5.167 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=6.37e-04 +Train: Iter=600 (0.073243 effective passes) train_loss=4.714 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=6.65e-04 +Train: Iter=650 (0.079347 effective passes) train_loss=5.089 iter_dt=2.82e-01s lr=3.00e-01 +effective_lr=6.84e-04 +Train: Iter=700 (0.085451 effective passes) train_loss=4.903 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=7.02e-04 +Train: Iter=750 (0.091554 effective passes) train_loss=4.889 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=7.19e-04 +Train: Iter=800 (0.097658 effective passes) train_loss=4.703 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=7.35e-04 +Train: Iter=850 (0.103761 effective passes) train_loss=4.628 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=7.49e-04 +Train: Iter=900 (0.109865 effective passes) train_loss=4.555 iter_dt=2.82e-01s lr=3.00e-01 +effective_lr=7.62e-04 +Train: Iter=950 (0.115969 effective passes) train_loss=4.634 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=7.75e-04 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.634 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=7.87e-04 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.549 val_pp=94.495 val_acc=0.270250 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.658 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=7.98e-04 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.465 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=8.08e-04 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.493 iter_dt=2.82e-01s lr=3.00e-01 +effective_lr=8.18e-04 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.409 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=8.27e-04 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.435 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=8.36e-04 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.468 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=8.44e-04 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.497 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=8.52e-04 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.495 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=8.59e-04 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.357 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=8.66e-04 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.427 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=8.72e-04 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.362 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=8.78e-04 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.621 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=8.84e-04 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.194 iter_dt=2.84e-01s lr=3.00e-01 +effective_lr=8.89e-04 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.556 iter_dt=2.85e-01s lr=3.00e-01 +effective_lr=8.95e-04 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.221 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=8.99e-04 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.261 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.04e-04 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.554 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.08e-04 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.145 iter_dt=2.78e-01s lr=3.00e-01 +effective_lr=9.12e-04 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.103 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.16e-04 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.295 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.20e-04 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.271 val_pp=71.619 val_acc=0.292839 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.288 iter_dt=2.78e-01s lr=3.00e-01 +effective_lr=9.24e-04 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.297 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.27e-04 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.258 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=9.30e-04 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.241 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.33e-04 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.300 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.36e-04 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.996 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.38e-04 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.180 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.41e-04 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.189 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.43e-04 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.184 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.46e-04 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.447 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.48e-04 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.223 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.50e-04 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.001 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.52e-04 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.434 iter_dt=2.82e-01s lr=3.00e-01 +effective_lr=9.54e-04 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.306 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.56e-04 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.199 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=9.57e-04 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.053 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.59e-04 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.374 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.60e-04 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.287 iter_dt=2.78e-01s lr=3.00e-01 +effective_lr=9.62e-04 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.222 iter_dt=2.78e-01s lr=3.00e-01 +effective_lr=9.63e-04 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.056 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.64e-04 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.242 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=9.66e-04 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.862 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.67e-04 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.103 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.68e-04 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.169 iter_dt=2.78e-01s lr=3.00e-01 +effective_lr=9.69e-04 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.232 iter_dt=2.78e-01s lr=3.00e-01 +effective_lr=9.70e-04 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.108 iter_dt=2.83e-01s lr=3.00e-01 +effective_lr=9.71e-04 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.330 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=9.72e-04 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.034 iter_dt=2.82e-01s lr=3.00e-01 +effective_lr=9.73e-04 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.003 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=9.74e-04 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.111 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.74e-04 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.170 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.75e-04 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.239 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.76e-04 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.199 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.76e-04 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.067 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=9.77e-04 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.052 iter_dt=2.85e-01s lr=3.00e-01 +effective_lr=9.78e-04 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.165 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.78e-04 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.988 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.79e-04 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.088 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.79e-04 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.148 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.80e-04 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.189 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.80e-04 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.080 iter_dt=2.78e-01s lr=3.00e-01 +effective_lr=9.81e-04 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.091 val_pp=59.827 val_acc=0.307278 + +[sophiag seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt sophiag --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456 --lr 0.0003 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sophiag', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SophiaG ( +Parameter Group 0 + betas: (0.9, 0.999) + capturable: False + lr: 0.0003 + maximize: False + rho: 0.04 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + capturable: False + lr: 0.0003 + maximize: False + rho: 0.04 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.758 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.832 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.463 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.363 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.193 iter_dt=3.23e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.090 val_pp=441.610 val_acc=0.157600 +Train: Iter=300 (0.036622 effective passes) train_loss=5.865 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=350 (0.042725 effective passes) train_loss=5.999 iter_dt=3.22e-01s lr=3.00e-04 +Train: Iter=400 (0.048829 effective passes) train_loss=5.706 iter_dt=3.25e-01s lr=3.00e-04 +Train: Iter=450 (0.054932 effective passes) train_loss=5.501 iter_dt=3.27e-01s lr=3.00e-04 +Train: Iter=500 (0.061036 effective passes) train_loss=5.394 iter_dt=3.20e-01s lr=3.00e-04 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.460 val_pp=235.157 val_acc=0.195499 +Train: Iter=550 (0.067140 effective passes) train_loss=5.557 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=600 (0.073243 effective passes) train_loss=5.029 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=650 (0.079347 effective passes) train_loss=5.469 iter_dt=3.21e-01s lr=3.00e-04 +Train: Iter=700 (0.085451 effective passes) train_loss=5.242 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=750 (0.091554 effective passes) train_loss=5.187 iter_dt=3.22e-01s lr=3.00e-04 +Train: Iter=800 (0.097658 effective passes) train_loss=5.003 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=850 (0.103761 effective passes) train_loss=4.944 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=900 (0.109865 effective passes) train_loss=4.851 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=950 (0.115969 effective passes) train_loss=4.867 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.880 iter_dt=3.19e-01s lr=3.00e-04 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.813 val_pp=123.039 val_acc=0.248173 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.919 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.722 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.695 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.631 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.640 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.688 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.674 iter_dt=3.21e-01s lr=3.00e-04 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.669 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.533 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.651 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.531 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.817 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.382 iter_dt=3.21e-01s lr=3.00e-04 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.727 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.385 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.418 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.718 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.302 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.304 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.465 iter_dt=3.20e-01s lr=3.00e-04 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.431 val_pp=83.999 val_acc=0.281746 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.483 iter_dt=3.23e-01s lr=3.00e-04 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.440 iter_dt=3.24e-01s lr=3.00e-04 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.422 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.381 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.438 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.153 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.330 iter_dt=3.29e-01s lr=3.00e-04 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.341 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.315 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.579 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.383 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.137 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.554 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.436 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.342 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.181 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.485 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.411 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.345 iter_dt=3.28e-01s lr=3.00e-04 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.192 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.365 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.958 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.230 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.291 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.367 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.249 iter_dt=3.21e-01s lr=3.00e-04 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.427 iter_dt=3.22e-01s lr=3.00e-04 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.154 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.117 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.222 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.280 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.384 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.338 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.184 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.193 iter_dt=3.21e-01s lr=3.00e-04 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.309 iter_dt=3.24e-01s lr=3.00e-04 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.105 iter_dt=3.25e-01s lr=3.00e-04 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.204 iter_dt=3.29e-01s lr=3.00e-04 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.273 iter_dt=3.26e-01s lr=3.00e-04 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.323 iter_dt=3.21e-01s lr=3.00e-04 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.197 iter_dt=3.19e-01s lr=3.00e-04 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.210 val_pp=67.352 val_acc=0.299429 + +[adopt seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adopt --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456 --lr 0.0003 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adopt', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +ADOPT ( +Parameter Group 0 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.0003 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.0003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.382 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.561 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.227 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.155 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=5.967 iter_dt=2.74e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.867 val_pp=353.188 val_acc=0.180328 +Train: Iter=300 (0.036622 effective passes) train_loss=5.785 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=350 (0.042725 effective passes) train_loss=5.839 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=400 (0.048829 effective passes) train_loss=5.533 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=450 (0.054932 effective passes) train_loss=5.332 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=500 (0.061036 effective passes) train_loss=5.274 iter_dt=2.75e-01s lr=3.00e-04 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.325 val_pp=205.400 val_acc=0.211224 +Train: Iter=550 (0.067140 effective passes) train_loss=5.448 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=600 (0.073243 effective passes) train_loss=4.917 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=650 (0.079347 effective passes) train_loss=5.399 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=700 (0.085451 effective passes) train_loss=5.181 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=750 (0.091554 effective passes) train_loss=5.128 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=800 (0.097658 effective passes) train_loss=4.949 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=850 (0.103761 effective passes) train_loss=4.936 iter_dt=2.77e-01s lr=3.00e-04 +Train: Iter=900 (0.109865 effective passes) train_loss=4.869 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=950 (0.115969 effective passes) train_loss=4.884 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.860 iter_dt=2.74e-01s lr=3.00e-04 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.819 val_pp=123.854 val_acc=0.247356 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.908 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.737 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.682 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.625 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.635 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.701 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.667 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.649 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.524 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.645 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.512 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.821 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.368 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.720 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.374 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.410 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.700 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.301 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.281 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.447 iter_dt=2.74e-01s lr=3.00e-04 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.414 val_pp=82.565 val_acc=0.283892 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.461 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.419 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.397 iter_dt=2.76e-01s lr=3.00e-04 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.354 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.427 iter_dt=2.78e-01s lr=3.00e-04 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.153 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.326 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.339 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.314 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.561 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.372 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.118 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.519 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.418 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.312 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.170 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.465 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.384 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.301 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.154 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.346 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.928 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.215 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.271 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.344 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.203 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.400 iter_dt=2.80e-01s lr=3.00e-04 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.118 iter_dt=2.92e-01s lr=3.00e-04 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.076 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.177 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.256 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.327 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.289 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.140 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.148 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.268 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.073 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.163 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.219 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.268 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.154 iter_dt=2.73e-01s lr=3.00e-04 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.171 val_pp=64.765 val_acc=0.303391 + +[mars seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt mars --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456 --lr 0.0003 --mars_lr 0.0009 --beta1 0.8 --beta2 0.999 --mars_beta1 0.95 --mars_beta2 0.99 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'mars', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.0009, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +MARS ( +Parameter Group 0 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.0009 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.1 + weight_decay_1d: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.0009 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.0 + weight_decay_1d: 0.1 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.252 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.459 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.181 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.120 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=5.951 iter_dt=2.79e-01s lr=9.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.858 val_pp=349.918 val_acc=0.177553 +Train: Iter=300 (0.036622 effective passes) train_loss=5.710 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=350 (0.042725 effective passes) train_loss=5.803 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=400 (0.048829 effective passes) train_loss=5.515 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=450 (0.054932 effective passes) train_loss=5.306 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=500 (0.061036 effective passes) train_loss=5.255 iter_dt=2.81e-01s lr=9.00e-04 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.305 val_pp=201.331 val_acc=0.208384 +Train: Iter=550 (0.067140 effective passes) train_loss=5.417 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=600 (0.073243 effective passes) train_loss=4.906 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=650 (0.079347 effective passes) train_loss=5.362 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=700 (0.085451 effective passes) train_loss=5.145 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=750 (0.091554 effective passes) train_loss=5.087 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=800 (0.097658 effective passes) train_loss=4.916 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=850 (0.103761 effective passes) train_loss=4.882 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=900 (0.109865 effective passes) train_loss=4.808 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=950 (0.115969 effective passes) train_loss=4.826 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.817 iter_dt=2.80e-01s lr=9.00e-04 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.751 val_pp=115.662 val_acc=0.252459 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.843 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.664 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.633 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.570 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.590 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.631 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.611 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.607 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.484 iter_dt=2.83e-01s lr=9.00e-04 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.576 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.467 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.756 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.300 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.657 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.323 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.350 iter_dt=2.83e-01s lr=9.00e-04 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.645 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.236 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.222 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.392 iter_dt=2.94e-01s lr=9.00e-04 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.355 val_pp=77.860 val_acc=0.287682 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.407 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.368 iter_dt=2.83e-01s lr=9.00e-04 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.324 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.293 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.356 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.091 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.255 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.259 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.263 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.504 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.298 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.053 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.487 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.355 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.262 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.105 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.413 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.324 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.256 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.103 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.289 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.896 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.141 iter_dt=2.84e-01s lr=9.00e-04 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.210 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.270 iter_dt=2.87e-01s lr=9.00e-04 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.155 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.369 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.080 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.049 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.136 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.222 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.288 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.240 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.094 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.115 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.202 iter_dt=2.83e-01s lr=9.00e-04 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.025 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.113 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.172 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.230 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.095 iter_dt=2.83e-01s lr=9.00e-04 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.122 val_pp=61.670 val_acc=0.305550 + +[adafactor seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt adafactor --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456 --lr 0.0003 --beta1 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adafactor', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Adafactor ( +Parameter Group 0 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.0003 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.1 + +Parameter Group 1 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.0003 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.464 iter_dt=2.90e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.453 iter_dt=2.92e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.160 iter_dt=3.00e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.137 iter_dt=2.93e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.004 iter_dt=2.92e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.914 val_pp=370.206 val_acc=0.181087 +Train: Iter=300 (0.036622 effective passes) train_loss=5.816 iter_dt=2.96e-01s lr=3.00e-04 +Train: Iter=350 (0.042725 effective passes) train_loss=5.921 iter_dt=2.95e-01s lr=3.00e-04 +Train: Iter=400 (0.048829 effective passes) train_loss=5.620 iter_dt=2.95e-01s lr=3.00e-04 +Train: Iter=450 (0.054932 effective passes) train_loss=5.434 iter_dt=2.94e-01s lr=3.00e-04 +Train: Iter=500 (0.061036 effective passes) train_loss=5.331 iter_dt=2.94e-01s lr=3.00e-04 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.436 val_pp=229.608 val_acc=0.204878 +Train: Iter=550 (0.067140 effective passes) train_loss=5.562 iter_dt=2.91e-01s lr=3.00e-04 +Train: Iter=600 (0.073243 effective passes) train_loss=5.035 iter_dt=2.94e-01s lr=3.00e-04 +Train: Iter=650 (0.079347 effective passes) train_loss=5.509 iter_dt=2.93e-01s lr=3.00e-04 +Train: Iter=700 (0.085451 effective passes) train_loss=5.316 iter_dt=2.90e-01s lr=3.00e-04 +Train: Iter=750 (0.091554 effective passes) train_loss=5.292 iter_dt=2.91e-01s lr=3.00e-04 +Train: Iter=800 (0.097658 effective passes) train_loss=5.076 iter_dt=2.91e-01s lr=3.00e-04 +Train: Iter=850 (0.103761 effective passes) train_loss=5.093 iter_dt=2.93e-01s lr=3.00e-04 +Train: Iter=900 (0.109865 effective passes) train_loss=5.043 iter_dt=2.96e-01s lr=3.00e-04 +Train: Iter=950 (0.115969 effective passes) train_loss=5.051 iter_dt=2.95e-01s lr=3.00e-04 +Train: Iter=1000 (0.122072 effective passes) train_loss=5.027 iter_dt=3.04e-01s lr=3.00e-04 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=5.031 val_pp=153.055 val_acc=0.227949 +Train: Iter=1050 (0.128176 effective passes) train_loss=5.124 iter_dt=2.93e-01s lr=3.00e-04 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.976 iter_dt=2.95e-01s lr=3.00e-04 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.898 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.868 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.842 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.979 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.893 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.870 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.769 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.865 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.711 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1600 (0.195315 effective passes) train_loss=5.032 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.572 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.922 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.612 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.624 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.904 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.510 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.489 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.659 iter_dt=2.85e-01s lr=3.00e-04 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.612 val_pp=100.683 val_acc=0.264980 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.640 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.629 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.575 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.537 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.598 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.342 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.485 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.532 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.490 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.745 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.547 iter_dt=2.91e-01s lr=3.00e-04 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.303 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.680 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.601 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.474 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.326 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.651 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.570 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.471 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.330 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.506 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3100 (0.378424 effective passes) train_loss=4.097 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.369 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.427 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.496 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.385 iter_dt=2.89e-01s lr=3.00e-04 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.576 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.283 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.251 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.330 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.415 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.494 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.438 iter_dt=2.91e-01s lr=3.00e-04 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.291 iter_dt=2.89e-01s lr=3.00e-04 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.306 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.414 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.210 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.315 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.392 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.447 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.298 iter_dt=2.86e-01s lr=3.00e-04 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.321 val_pp=75.290 val_acc=0.289244 + +[lamb seed=0 lr_scale=27] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt lamb --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456 --lr 0.027 --beta1 0.9 --beta2 0.999 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.027, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lamb', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lamb ( +Parameter Group 0 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.027 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.027 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.723 iter_dt=2.80e-01s lr=2.70e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=6.794 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=6.423 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=6.328 iter_dt=2.85e-01s lr=2.70e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=6.140 iter_dt=2.81e-01s lr=2.70e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.028 val_pp=415.060 val_acc=0.168570 +Train: Iter=300 (0.036622 effective passes) train_loss=5.780 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=350 (0.042725 effective passes) train_loss=5.951 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=400 (0.048829 effective passes) train_loss=5.663 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=450 (0.054932 effective passes) train_loss=5.432 iter_dt=2.86e-01s lr=2.70e-02 +Train: Iter=500 (0.061036 effective passes) train_loss=5.352 iter_dt=2.83e-01s lr=2.70e-02 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.411 val_pp=223.751 val_acc=0.202606 +Train: Iter=550 (0.067140 effective passes) train_loss=5.519 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=600 (0.073243 effective passes) train_loss=4.987 iter_dt=2.84e-01s lr=2.70e-02 +Train: Iter=650 (0.079347 effective passes) train_loss=5.454 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=700 (0.085451 effective passes) train_loss=5.258 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=750 (0.091554 effective passes) train_loss=5.189 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=800 (0.097658 effective passes) train_loss=5.003 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=850 (0.103761 effective passes) train_loss=4.954 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=900 (0.109865 effective passes) train_loss=4.879 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=950 (0.115969 effective passes) train_loss=4.913 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.888 iter_dt=2.82e-01s lr=2.70e-02 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.842 val_pp=126.748 val_acc=0.246984 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.933 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.734 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.734 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.671 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.673 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.707 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.698 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.699 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.577 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.650 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.549 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.862 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.410 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.754 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.425 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.452 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.752 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.327 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.333 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.491 iter_dt=2.81e-01s lr=2.70e-02 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.456 val_pp=86.124 val_acc=0.279652 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.493 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.463 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.430 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.404 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.464 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.173 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.350 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.361 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.339 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.613 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.400 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.159 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.566 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.448 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.344 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.200 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.498 iter_dt=2.85e-01s lr=2.70e-02 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.428 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.357 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.209 iter_dt=2.88e-01s lr=2.70e-02 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.372 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.992 iter_dt=2.84e-01s lr=2.70e-02 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.250 iter_dt=2.84e-01s lr=2.70e-02 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.318 iter_dt=2.86e-01s lr=2.70e-02 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.374 iter_dt=2.87e-01s lr=2.70e-02 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.234 iter_dt=2.87e-01s lr=2.70e-02 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.441 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.182 iter_dt=2.87e-01s lr=2.70e-02 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.130 iter_dt=2.84e-01s lr=2.70e-02 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.239 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.325 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.380 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.334 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.181 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.202 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.306 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.122 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.215 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.268 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.328 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.197 iter_dt=2.84e-01s lr=2.70e-02 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.211 val_pp=67.393 val_acc=0.298441 + +[scion seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt scion --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456 --lr 0.0003 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +Scion ( +Parameter Group 0 + lr: 0.0003 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.0003 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.068 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.438 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.293 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.345 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.366 iter_dt=2.88e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.255 val_pp=520.603 val_acc=0.140072 +Train: Iter=300 (0.036622 effective passes) train_loss=6.215 iter_dt=2.91e-01s lr=3.00e-04 +Train: Iter=350 (0.042725 effective passes) train_loss=6.420 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=400 (0.048829 effective passes) train_loss=6.229 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=450 (0.054932 effective passes) train_loss=6.070 iter_dt=2.89e-01s lr=3.00e-04 +Train: Iter=500 (0.061036 effective passes) train_loss=6.009 iter_dt=2.87e-01s lr=3.00e-04 +>Eval: Iter=512 (0.062501 effective passes) val_loss=6.164 val_pp=475.152 val_acc=0.145535 +Train: Iter=550 (0.067140 effective passes) train_loss=6.303 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=600 (0.073243 effective passes) train_loss=5.813 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=650 (0.079347 effective passes) train_loss=6.311 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=700 (0.085451 effective passes) train_loss=6.090 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=750 (0.091554 effective passes) train_loss=6.104 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=800 (0.097658 effective passes) train_loss=6.024 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=850 (0.103761 effective passes) train_loss=6.034 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=900 (0.109865 effective passes) train_loss=5.985 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=950 (0.115969 effective passes) train_loss=5.963 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=1000 (0.122072 effective passes) train_loss=5.954 iter_dt=2.86e-01s lr=3.00e-04 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=6.059 val_pp=427.777 val_acc=0.148499 +Train: Iter=1050 (0.128176 effective passes) train_loss=6.140 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1100 (0.134279 effective passes) train_loss=6.035 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=1150 (0.140383 effective passes) train_loss=5.909 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=1200 (0.146487 effective passes) train_loss=5.938 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1250 (0.152590 effective passes) train_loss=6.019 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1300 (0.158694 effective passes) train_loss=6.086 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1350 (0.164797 effective passes) train_loss=5.989 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1400 (0.170901 effective passes) train_loss=6.012 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1450 (0.177005 effective passes) train_loss=5.943 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1500 (0.183108 effective passes) train_loss=6.140 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1550 (0.189212 effective passes) train_loss=5.924 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1600 (0.195315 effective passes) train_loss=6.259 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1650 (0.201419 effective passes) train_loss=5.887 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1700 (0.207523 effective passes) train_loss=6.295 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1750 (0.213626 effective passes) train_loss=5.973 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1800 (0.219730 effective passes) train_loss=5.930 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=1850 (0.225834 effective passes) train_loss=6.123 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=1900 (0.231937 effective passes) train_loss=5.895 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=1950 (0.238041 effective passes) train_loss=6.107 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2000 (0.244144 effective passes) train_loss=6.133 iter_dt=2.86e-01s lr=3.00e-04 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=6.075 val_pp=434.811 val_acc=0.149952 +Train: Iter=2050 (0.250248 effective passes) train_loss=6.210 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2100 (0.256352 effective passes) train_loss=6.103 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2150 (0.262455 effective passes) train_loss=6.146 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2200 (0.268559 effective passes) train_loss=5.926 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2250 (0.274662 effective passes) train_loss=5.991 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2300 (0.280766 effective passes) train_loss=6.081 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2350 (0.286870 effective passes) train_loss=5.899 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=2400 (0.292973 effective passes) train_loss=6.340 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2450 (0.299077 effective passes) train_loss=5.931 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2500 (0.305180 effective passes) train_loss=6.231 iter_dt=2.93e-01s lr=3.00e-04 +Train: Iter=2550 (0.311284 effective passes) train_loss=6.321 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2600 (0.317388 effective passes) train_loss=5.826 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2650 (0.323491 effective passes) train_loss=6.034 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2700 (0.329595 effective passes) train_loss=6.212 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2750 (0.335698 effective passes) train_loss=6.159 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2800 (0.341802 effective passes) train_loss=6.052 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2850 (0.347906 effective passes) train_loss=6.250 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2900 (0.354009 effective passes) train_loss=6.236 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2950 (0.360113 effective passes) train_loss=6.052 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3000 (0.366217 effective passes) train_loss=5.913 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=3050 (0.372320 effective passes) train_loss=6.193 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3100 (0.378424 effective passes) train_loss=5.563 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3150 (0.384527 effective passes) train_loss=5.994 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3200 (0.390631 effective passes) train_loss=5.992 iter_dt=2.91e-01s lr=3.00e-04 +Train: Iter=3250 (0.396735 effective passes) train_loss=6.134 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3300 (0.402838 effective passes) train_loss=6.180 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=3350 (0.408942 effective passes) train_loss=6.064 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=3400 (0.415045 effective passes) train_loss=5.894 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=3450 (0.421149 effective passes) train_loss=5.831 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3500 (0.427253 effective passes) train_loss=5.956 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=3550 (0.433356 effective passes) train_loss=6.192 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3600 (0.439460 effective passes) train_loss=6.262 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3650 (0.445563 effective passes) train_loss=6.175 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3700 (0.451667 effective passes) train_loss=6.016 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3750 (0.457771 effective passes) train_loss=6.121 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3800 (0.463874 effective passes) train_loss=6.448 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3850 (0.469978 effective passes) train_loss=5.962 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3900 (0.476081 effective passes) train_loss=5.912 iter_dt=2.90e-01s lr=3.00e-04 +Train: Iter=3950 (0.482185 effective passes) train_loss=6.080 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=4000 (0.488289 effective passes) train_loss=6.398 iter_dt=2.93e-01s lr=3.00e-04 +Train: Iter=4050 (0.494392 effective passes) train_loss=5.952 iter_dt=2.88e-01s lr=3.00e-04 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=6.072 val_pp=433.616 val_acc=0.151756 + +[scion-light seed=0 lr_scale=0.3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt scion-light --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456 --lr 0.0003 --momentum 0.9 --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion-light', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +ScionLight ( +Parameter Group 0 + lr: 0.0003 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.0003 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.337 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.690 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.537 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.580 iter_dt=2.94e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.598 iter_dt=2.91e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.452 val_pp=634.052 val_acc=0.142305 +Train: Iter=300 (0.036622 effective passes) train_loss=6.625 iter_dt=2.90e-01s lr=3.00e-04 +Train: Iter=350 (0.042725 effective passes) train_loss=6.661 iter_dt=2.99e-01s lr=3.00e-04 +Train: Iter=400 (0.048829 effective passes) train_loss=6.425 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=450 (0.054932 effective passes) train_loss=6.259 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=500 (0.061036 effective passes) train_loss=6.269 iter_dt=2.86e-01s lr=3.00e-04 +>Eval: Iter=512 (0.062501 effective passes) val_loss=6.336 val_pp=564.683 val_acc=0.145407 +Train: Iter=550 (0.067140 effective passes) train_loss=6.603 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=600 (0.073243 effective passes) train_loss=5.972 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=650 (0.079347 effective passes) train_loss=6.548 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=700 (0.085451 effective passes) train_loss=6.388 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=750 (0.091554 effective passes) train_loss=6.350 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=800 (0.097658 effective passes) train_loss=6.264 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=850 (0.103761 effective passes) train_loss=6.295 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=900 (0.109865 effective passes) train_loss=6.292 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=950 (0.115969 effective passes) train_loss=6.261 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1000 (0.122072 effective passes) train_loss=6.169 iter_dt=2.85e-01s lr=3.00e-04 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=6.414 val_pp=610.194 val_acc=0.142185 +Train: Iter=1050 (0.128176 effective passes) train_loss=6.393 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1100 (0.134279 effective passes) train_loss=6.306 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=1150 (0.140383 effective passes) train_loss=6.124 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=1200 (0.146487 effective passes) train_loss=6.160 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=1250 (0.152590 effective passes) train_loss=6.254 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1300 (0.158694 effective passes) train_loss=6.348 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1350 (0.164797 effective passes) train_loss=6.198 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1400 (0.170901 effective passes) train_loss=6.329 iter_dt=2.89e-01s lr=3.00e-04 +Train: Iter=1450 (0.177005 effective passes) train_loss=6.241 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1500 (0.183108 effective passes) train_loss=6.430 iter_dt=2.89e-01s lr=3.00e-04 +Train: Iter=1550 (0.189212 effective passes) train_loss=6.203 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1600 (0.195315 effective passes) train_loss=6.490 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1650 (0.201419 effective passes) train_loss=6.133 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1700 (0.207523 effective passes) train_loss=6.648 iter_dt=2.92e-01s lr=3.00e-04 +Train: Iter=1750 (0.213626 effective passes) train_loss=6.291 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=1800 (0.219730 effective passes) train_loss=6.180 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1850 (0.225834 effective passes) train_loss=6.390 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1900 (0.231937 effective passes) train_loss=6.212 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1950 (0.238041 effective passes) train_loss=6.375 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=2000 (0.244144 effective passes) train_loss=6.475 iter_dt=2.88e-01s lr=3.00e-04 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=6.294 val_pp=541.089 val_acc=0.146240 +Train: Iter=2050 (0.250248 effective passes) train_loss=6.552 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2100 (0.256352 effective passes) train_loss=6.330 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=2150 (0.262455 effective passes) train_loss=6.417 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2200 (0.268559 effective passes) train_loss=6.074 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=2250 (0.274662 effective passes) train_loss=6.178 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=2300 (0.280766 effective passes) train_loss=6.254 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=2350 (0.286870 effective passes) train_loss=6.077 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=2400 (0.292973 effective passes) train_loss=6.600 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=2450 (0.299077 effective passes) train_loss=6.104 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=2500 (0.305180 effective passes) train_loss=6.455 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=2550 (0.311284 effective passes) train_loss=6.559 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=2600 (0.317388 effective passes) train_loss=6.030 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=2650 (0.323491 effective passes) train_loss=6.160 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=2700 (0.329595 effective passes) train_loss=6.359 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=2750 (0.335698 effective passes) train_loss=6.320 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=2800 (0.341802 effective passes) train_loss=6.246 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=2850 (0.347906 effective passes) train_loss=6.442 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=2900 (0.354009 effective passes) train_loss=6.435 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=2950 (0.360113 effective passes) train_loss=6.248 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=3000 (0.366217 effective passes) train_loss=6.096 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=3050 (0.372320 effective passes) train_loss=6.414 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3100 (0.378424 effective passes) train_loss=5.788 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=3150 (0.384527 effective passes) train_loss=6.209 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3200 (0.390631 effective passes) train_loss=6.203 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3250 (0.396735 effective passes) train_loss=6.362 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=3300 (0.402838 effective passes) train_loss=6.462 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3350 (0.408942 effective passes) train_loss=6.288 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=3400 (0.415045 effective passes) train_loss=6.100 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=3450 (0.421149 effective passes) train_loss=6.030 iter_dt=2.82e-01s lr=3.00e-04 +Train: Iter=3500 (0.427253 effective passes) train_loss=6.156 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=3550 (0.433356 effective passes) train_loss=6.399 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=3600 (0.439460 effective passes) train_loss=6.482 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=3650 (0.445563 effective passes) train_loss=6.377 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=3700 (0.451667 effective passes) train_loss=6.222 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=3750 (0.457771 effective passes) train_loss=6.341 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=3800 (0.463874 effective passes) train_loss=6.652 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3850 (0.469978 effective passes) train_loss=6.156 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3900 (0.476081 effective passes) train_loss=6.089 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=3950 (0.482185 effective passes) train_loss=6.243 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=4000 (0.488289 effective passes) train_loss=6.564 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=4050 (0.494392 effective passes) train_loss=6.104 iter_dt=2.85e-01s lr=3.00e-04 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=6.240 val_pp=512.879 val_acc=0.148157 + +[d-muon seed=0 lr_scale=3] /root/miniconda3/bin/torchrun --standalone --nproc_per_node=1 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --distributed_backend nccl --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt d-muon --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456 --lr 0.003 --beta1 0.8 --beta2 0.999 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': 'nccl', 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'd-muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Distributed DataReader Initialized for Worker 0/1 +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +DistributedMuon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.003 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.1 + +Parameter Group 1 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.003 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.593 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=5.888 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=5.545 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.357 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.291 iter_dt=2.81e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.120 val_pp=167.299 val_acc=0.216290 +Train: Iter=300 (0.036622 effective passes) train_loss=4.916 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.066 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=4.672 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=4.495 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=4.534 iter_dt=2.80e-01s lr=3.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=4.620 val_pp=101.444 val_acc=0.263233 +Train: Iter=550 (0.067140 effective passes) train_loss=4.581 iter_dt=2.83e-01s lr=3.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.359 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=4.739 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.622 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.613 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.484 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.389 iter_dt=2.83e-01s lr=3.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.356 iter_dt=2.83e-01s lr=3.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.454 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.481 iter_dt=2.81e-01s lr=3.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.364 val_pp=78.575 val_acc=0.284447 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.503 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.313 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.329 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.257 iter_dt=2.83e-01s lr=3.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.288 iter_dt=2.85e-01s lr=3.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.339 iter_dt=2.84e-01s lr=3.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.355 iter_dt=2.88e-01s lr=3.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.381 iter_dt=2.83e-01s lr=3.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.247 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.321 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.259 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.524 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.103 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.459 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.129 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.163 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.475 iter_dt=3.00e-01s lr=3.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.055 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.012 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.220 iter_dt=2.81e-01s lr=3.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.190 val_pp=65.992 val_acc=0.298725 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.184 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.216 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.191 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.186 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.213 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.925 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.117 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.113 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.109 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.376 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.151 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.948 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.361 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.248 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.136 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.993 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.300 iter_dt=2.83e-01s lr=3.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.215 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.164 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.018 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.192 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.819 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.039 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.129 iter_dt=2.84e-01s lr=3.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.188 iter_dt=2.91e-01s lr=3.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.056 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.270 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.013 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.964 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.054 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.127 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.227 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.152 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.040 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.023 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.122 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.961 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.064 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.123 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.151 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.062 iter_dt=2.82e-01s lr=3.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.057 val_pp=57.809 val_acc=0.309958 + +[muon-pytorch seed=0 lr_scale=3] /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/src/main.py --config_format base --model llama --dataset token-bin --datasets_dir /root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets --device cuda:0 --n_layer 8 --n_head 6 --n_embd 384 --batch_size 16 --acc_steps 8 --sequence_length 512 --opt muon-pytorch --train_token_budget 268435456 --eval_at_tokens 16777216 33554432 67108864 134217728 268435456 --strict_sub_one_pass --fixed_data_boundaries --lazy_data_permutation --scheduler warmup_constant --warmup_steps 16 --eval_interval 0 --eval_batches 64 --limit_final_eval --latest_ckpt_interval 4096 --log_interval 50 --weight_decay 0.1 --grad_clip 0.5 --seed 0 --data_seed 1337 --results_base_folder /root/autodl-tmp/llm-optimizer-results/formal --experiment_name sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456 --lr 0.06 --muon_adamw_lr 0.003 --momentum 0.95 --nesterov True --train_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin --val_data_path /root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin +Starting Experiment: sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.06, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon-pytorch', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.003, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: + +Train: Iter=50 (0.006104 effective passes) train_loss=6.777 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=5.844 iter_dt=2.83e-01s lr=6.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=5.551 iter_dt=2.85e-01s lr=6.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=5.476 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=5.435 iter_dt=2.82e-01s lr=6.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.278 val_pp=195.998 val_acc=0.204409 +Train: Iter=300 (0.036622 effective passes) train_loss=5.093 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=350 (0.042725 effective passes) train_loss=5.293 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=400 (0.048829 effective passes) train_loss=4.968 iter_dt=2.85e-01s lr=6.00e-02 +Train: Iter=450 (0.054932 effective passes) train_loss=4.828 iter_dt=2.81e-01s lr=6.00e-02 +Train: Iter=500 (0.061036 effective passes) train_loss=4.875 iter_dt=2.88e-01s lr=6.00e-02 +>Eval: Iter=512 (0.062501 effective passes) val_loss=4.949 val_pp=141.029 val_acc=0.237040 +Train: Iter=550 (0.067140 effective passes) train_loss=4.966 iter_dt=2.86e-01s lr=6.00e-02 +Train: Iter=600 (0.073243 effective passes) train_loss=4.692 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=650 (0.079347 effective passes) train_loss=5.086 iter_dt=2.84e-01s lr=6.00e-02 +Train: Iter=700 (0.085451 effective passes) train_loss=4.971 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=750 (0.091554 effective passes) train_loss=5.011 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=800 (0.097658 effective passes) train_loss=4.838 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=850 (0.103761 effective passes) train_loss=4.748 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=900 (0.109865 effective passes) train_loss=4.758 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=950 (0.115969 effective passes) train_loss=4.799 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.851 iter_dt=2.79e-01s lr=6.00e-02 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.762 val_pp=116.974 val_acc=0.250929 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.860 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.709 iter_dt=2.77e-01s lr=6.00e-02 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.698 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.659 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.679 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.745 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.736 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.738 iter_dt=2.78e-01s lr=6.00e-02 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.616 iter_dt=2.93e-01s lr=6.00e-02 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.733 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.621 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.924 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.464 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.849 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.500 iter_dt=2.81e-01s lr=6.00e-02 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.541 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.826 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.456 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.470 iter_dt=2.81e-01s lr=6.00e-02 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.599 iter_dt=2.79e-01s lr=6.00e-02 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.571 val_pp=96.684 val_acc=0.266104 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.601 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.601 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.575 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.538 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.580 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.313 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.486 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.528 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.493 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.764 iter_dt=2.78e-01s lr=6.00e-02 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.560 iter_dt=2.78e-01s lr=6.00e-02 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.319 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.730 iter_dt=2.78e-01s lr=6.00e-02 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.609 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.494 iter_dt=2.81e-01s lr=6.00e-02 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.375 iter_dt=2.81e-01s lr=6.00e-02 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.680 iter_dt=2.81e-01s lr=6.00e-02 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.600 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.538 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.373 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.540 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3100 (0.378424 effective passes) train_loss=4.161 iter_dt=2.81e-01s lr=6.00e-02 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.434 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.516 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.582 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.490 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.673 iter_dt=2.81e-01s lr=6.00e-02 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.392 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.339 iter_dt=2.86e-01s lr=6.00e-02 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.443 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.542 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.599 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.563 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.402 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.419 iter_dt=2.77e-01s lr=6.00e-02 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.546 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.348 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.428 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.492 iter_dt=2.78e-01s lr=6.00e-02 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.593 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.447 iter_dt=2.79e-01s lr=6.00e-02 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.457 val_pp=86.244 val_acc=0.274332 +$ /root/miniconda3/bin/python3.12 /root/autodl-tmp/llm-optimizer-benchmark/scripts/sub_one_pass/summarize.py /root/autodl-tmp/llm-optimizer-results/formal +Wrote 182 rows to /root/autodl-tmp/llm-optimizer-results/formal/sub_one_pass_results.csv diff --git a/exps/sub_one_pass/formal/manifest_confirm.json b/exps/sub_one_pass/formal/manifest_confirm.json new file mode 100644 index 0000000..b6b5438 --- /dev/null +++ b/exps/sub_one_pass/formal/manifest_confirm.json @@ -0,0 +1,766 @@ +{ + "protocol": "sub-one-pass", + "stage": "confirm", + "optimizers": [ + "adamw", + "ademamix", + "soap", + "muon" + ], + "invocations": [ + { + "arguments": { + "optimizers": [ + "adamw", + "ademamix", + "soap", + "muon" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 1, + 2 + ], + "data_seed": 1337, + "stage": "confirm", + "eval_tokens": [ + 16777216, + 33554432, + 67108864, + 134217728, + 268435456 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 1.0 + ], + "lr_scale_file": "/root/autodl-tmp/llm-optimizer-results/formal/selected_lr_scales.json", + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785263889.9930491 + } + ], + "runs": [ + { + "optimizer": "adamw", + "seed": 1, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "adamw", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "1", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456", + "--lr", + "0.001", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456", + "status": "completed", + "started_at_unix": 1785263889.993321, + "returncode": 0, + "finished_at_unix": 1785265013.7214384 + }, + { + "optimizer": "ademamix", + "seed": 1, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "ademamix", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "1", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456", + "--lr", + "0.001", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--adema_beta3", + "0.999", + "--adema_alpha", + "8.0", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "--adema_beta3_warmup", + "4096", + "--adema_alpha_warmup", + "4096" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456", + "status": "completed", + "started_at_unix": 1785265013.7221003, + "returncode": 0, + "finished_at_unix": 1785266153.086236 + }, + { + "optimizer": "soap", + "seed": 1, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "soap", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "1", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456", + "--lr", + "0.001", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456", + "status": "completed", + "started_at_unix": 1785266153.0867734, + "returncode": 0, + "finished_at_unix": 1785267443.7371788 + }, + { + "optimizer": "muon", + "seed": 1, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "muon", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "1", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456", + "--lr", + "0.003", + "--muon_lr_factor", + "0.03", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--momentum", + "0.95", + "--nesterov", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456", + "status": "completed", + "started_at_unix": 1785267443.738026, + "returncode": 0, + "finished_at_unix": 1785268618.1433387 + }, + { + "optimizer": "adamw", + "seed": 2, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "adamw", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "2", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456", + "--lr", + "0.001", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456", + "status": "completed", + "started_at_unix": 1785268618.143948, + "returncode": 0, + "finished_at_unix": 1785269747.917626 + }, + { + "optimizer": "ademamix", + "seed": 2, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "ademamix", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "2", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456", + "--lr", + "0.001", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--adema_beta3", + "0.999", + "--adema_alpha", + "8.0", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "--adema_beta3_warmup", + "4096", + "--adema_alpha_warmup", + "4096" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456", + "status": "completed", + "started_at_unix": 1785269747.9182587, + "returncode": 0, + "finished_at_unix": 1785270883.0380874 + }, + { + "optimizer": "soap", + "seed": 2, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "soap", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "2", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456", + "--lr", + "0.001", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456", + "status": "completed", + "started_at_unix": 1785270883.0387654, + "returncode": 0, + "finished_at_unix": 1785272177.0408113 + }, + { + "optimizer": "muon", + "seed": 2, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "muon", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "2", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456", + "--lr", + "0.003", + "--muon_lr_factor", + "0.03", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--momentum", + "0.95", + "--nesterov", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456", + "status": "completed", + "started_at_unix": 1785272177.0415087, + "returncode": 0, + "finished_at_unix": 1785273368.245155 + } + ] +} diff --git a/exps/sub_one_pass/formal/manifest_main.json b/exps/sub_one_pass/formal/manifest_main.json new file mode 100644 index 0000000..4b55aaf --- /dev/null +++ b/exps/sub_one_pass/formal/manifest_main.json @@ -0,0 +1,1778 @@ +{ + "protocol": "sub-one-pass", + "stage": "main", + "optimizers": [ + "adamw", + "sgd", + "muon", + "soap", + "ademamix", + "lion", + "sf-adamw", + "sf-sgd", + "signsgd", + "signum", + "prodigy", + "sophiag", + "adopt", + "mars", + "adafactor", + "lamb", + "scion", + "scion-light", + "d-muon", + "muon-pytorch" + ], + "invocations": [ + { + "arguments": { + "optimizers": [ + "all" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "main", + "eval_tokens": [ + 16777216, + 33554432, + 67108864, + 134217728, + 268435456 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 1.0 + ], + "lr_scale_file": "/root/autodl-tmp/llm-optimizer-results/formal/selected_lr_scales.json", + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785240733.722941 + } + ], + "runs": [ + { + "optimizer": "adamw", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "adamw", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456", + "--lr", + "0.001", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456", + "status": "completed", + "started_at_unix": 1785240733.7232316, + "returncode": 0, + "finished_at_unix": 1785241845.113231 + }, + { + "optimizer": "sgd", + "seed": 0, + "lr_scale": 9.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sgd", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456", + "--lr", + "0.09", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456", + "status": "completed", + "started_at_unix": 1785241845.113773, + "returncode": 0, + "finished_at_unix": 1785242952.117789 + }, + { + "optimizer": "muon", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "muon", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456", + "--lr", + "0.003", + "--muon_lr_factor", + "0.03", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--momentum", + "0.95", + "--nesterov", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456", + "status": "completed", + "started_at_unix": 1785242952.118285, + "returncode": 0, + "finished_at_unix": 1785244138.8960078 + }, + { + "optimizer": "soap", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "soap", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456", + "--lr", + "0.001", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456", + "status": "completed", + "started_at_unix": 1785244138.8966284, + "returncode": 0, + "finished_at_unix": 1785245435.5588088 + }, + { + "optimizer": "ademamix", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "ademamix", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456", + "--lr", + "0.001", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--adema_beta3", + "0.999", + "--adema_alpha", + "8.0", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "--adema_beta3_warmup", + "4096", + "--adema_alpha_warmup", + "4096" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456", + "status": "completed", + "started_at_unix": 1785245435.559418, + "returncode": 0, + "finished_at_unix": 1785246575.0341396 + }, + { + "optimizer": "lion", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "lion", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456", + "--lr", + "0.0003", + "--beta1", + "0.9", + "--beta2", + "0.99", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456", + "status": "completed", + "started_at_unix": 1785246575.0347776, + "returncode": 0, + "finished_at_unix": 1785247704.3608232 + }, + { + "optimizer": "sf-adamw", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sf-adamw", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456", + "--lr", + "0.001", + "--beta1", + "0.9", + "--beta2", + "0.9999", + "--scheduler", + "none", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456", + "status": "completed", + "started_at_unix": 1785247704.3615396, + "returncode": 0, + "finished_at_unix": 1785248833.3120282 + }, + { + "optimizer": "sf-sgd", + "seed": 0, + "lr_scale": 729.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sf-sgd", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456", + "--lr", + "7.29", + "--momentum", + "0.9", + "--scheduler", + "none", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456", + "status": "completed", + "started_at_unix": 1785248833.312737, + "returncode": 0, + "finished_at_unix": 1785249953.7626507 + }, + { + "optimizer": "signsgd", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "signsgd", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456", + "--lr", + "0.0003", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456", + "status": "completed", + "started_at_unix": 1785249953.763408, + "returncode": 0, + "finished_at_unix": 1785251077.6532207 + }, + { + "optimizer": "signum", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "signum", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456", + "--lr", + "0.001", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456", + "status": "completed", + "started_at_unix": 1785251077.6540325, + "returncode": 0, + "finished_at_unix": 1785252208.608245 + }, + { + "optimizer": "prodigy", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "prodigy", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456", + "--lr", + "0.3", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--prodigy_use_bias_correction", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456", + "status": "completed", + "started_at_unix": 1785252208.60905, + "returncode": 0, + "finished_at_unix": 1785253367.7907028 + }, + { + "optimizer": "sophiag", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sophiag", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456", + "--lr", + "0.0003", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456", + "status": "completed", + "started_at_unix": 1785253367.7918584, + "returncode": 0, + "finished_at_unix": 1785254526.6000762 + }, + { + "optimizer": "adopt", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "adopt", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456", + "--lr", + "0.0003", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456", + "status": "completed", + "started_at_unix": 1785254526.6009305, + "returncode": 0, + "finished_at_unix": 1785255662.1402504 + }, + { + "optimizer": "mars", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "mars", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456", + "--lr", + "0.0003", + "--mars_lr", + "0.0009", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--mars_beta1", + "0.95", + "--mars_beta2", + "0.99", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456", + "status": "completed", + "started_at_unix": 1785255662.1411412, + "returncode": 0, + "finished_at_unix": 1785256824.1004772 + }, + { + "optimizer": "adafactor", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "adafactor", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456", + "--lr", + "0.0003", + "--beta1", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456", + "status": "completed", + "started_at_unix": 1785256824.101416, + "returncode": 0, + "finished_at_unix": 1785258017.0072856 + }, + { + "optimizer": "lamb", + "seed": 0, + "lr_scale": 27.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "lamb", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456", + "--lr", + "0.027", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456", + "status": "completed", + "started_at_unix": 1785258017.0082624, + "returncode": 0, + "finished_at_unix": 1785259186.381071 + }, + { + "optimizer": "scion", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "scion", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456", + "--lr", + "0.0003", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456", + "status": "completed", + "started_at_unix": 1785259186.3820555, + "returncode": 0, + "finished_at_unix": 1785260373.767707 + }, + { + "optimizer": "scion-light", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "scion-light", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456", + "--lr", + "0.0003", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456", + "status": "completed", + "started_at_unix": 1785260373.7687235, + "returncode": 0, + "finished_at_unix": 1785261555.4226208 + }, + { + "optimizer": "d-muon", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/torchrun", + "--standalone", + "--nproc_per_node=1", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--distributed_backend", + "nccl", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "d-muon", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456", + "--lr", + "0.003", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--momentum", + "0.95", + "--nesterov", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456", + "status": "completed", + "started_at_unix": 1785261555.423677, + "returncode": 0, + "finished_at_unix": 1785262722.6424353 + }, + { + "optimizer": "muon-pytorch", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "muon-pytorch", + "--train_token_budget", + "268435456", + "--eval_at_tokens", + "16777216", + "33554432", + "67108864", + "134217728", + "268435456", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "4096", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456", + "--lr", + "0.06", + "--muon_adamw_lr", + "0.003", + "--momentum", + "0.95", + "--nesterov", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456", + "status": "completed", + "started_at_unix": 1785262722.6434212, + "returncode": 0, + "finished_at_unix": 1785263883.4229589 + } + ] +} diff --git a/exps/sub_one_pass/formal/manifest_tune.json b/exps/sub_one_pass/formal/manifest_tune.json new file mode 100644 index 0000000..3048d4e --- /dev/null +++ b/exps/sub_one_pass/formal/manifest_tune.json @@ -0,0 +1,7540 @@ +{ + "protocol": "sub-one-pass", + "stage": "tune", + "optimizers": [ + "adamw", + "sgd", + "muon", + "soap", + "ademamix", + "lion", + "sf-adamw", + "sf-sgd", + "signsgd", + "signum", + "prodigy", + "sophiag", + "adopt", + "mars", + "adafactor", + "lamb", + "scion", + "scion-light", + "d-muon", + "muon-pytorch" + ], + "invocations": [ + { + "arguments": { + "optimizers": [ + "all" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 0.3, + 1.0, + 3.0 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785232151.4482641 + }, + { + "arguments": { + "optimizers": [ + "adafactor" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 0.09999999999999999 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785236958.2033532 + }, + { + "arguments": { + "optimizers": [ + "adopt" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 0.09999999999999999 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785237039.9088614 + }, + { + "arguments": { + "optimizers": [ + "d-muon" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 9.0 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785237118.2140477 + }, + { + "arguments": { + "optimizers": [ + "lamb" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 9.0 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785237200.1728685 + }, + { + "arguments": { + "optimizers": [ + "lion" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 0.09999999999999999 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785237280.3405185 + }, + { + "arguments": { + "optimizers": [ + "mars" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 0.09999999999999999 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785237359.69845 + }, + { + "arguments": { + "optimizers": [ + "muon" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 9.0 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785237439.9371557 + }, + { + "arguments": { + "optimizers": [ + "muon-pytorch" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 9.0 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785237520.9196365 + }, + { + "arguments": { + "optimizers": [ + "prodigy" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 0.09999999999999999 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785237600.6580088 + }, + { + "arguments": { + "optimizers": [ + "scion" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 0.09999999999999999 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785237681.0706282 + }, + { + "arguments": { + "optimizers": [ + "scion-light" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 0.09999999999999999 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785237762.3963077 + }, + { + "arguments": { + "optimizers": [ + "sf-sgd" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 9.0 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785237843.540629 + }, + { + "arguments": { + "optimizers": [ + "sgd" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 9.0 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785237921.6784637 + }, + { + "arguments": { + "optimizers": [ + "signsgd" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 0.09999999999999999 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785237999.969892 + }, + { + "arguments": { + "optimizers": [ + "sophiag" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 0.09999999999999999 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785238078.8757858 + }, + { + "arguments": { + "optimizers": [ + "lamb" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 27.0 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785238159.479466 + }, + { + "arguments": { + "optimizers": [ + "sf-sgd" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 27.0 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785238240.1710327 + }, + { + "arguments": { + "optimizers": [ + "sgd" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 27.0 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785238318.308832 + }, + { + "arguments": { + "optimizers": [ + "all" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 0.3, + 1.0, + 3.0 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785240132.2373602 + }, + { + "arguments": { + "optimizers": [ + "lamb" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 81.0 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785240132.4783885 + }, + { + "arguments": { + "optimizers": [ + "sf-sgd" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 81.0 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785240213.3180578 + }, + { + "arguments": { + "optimizers": [ + "sf-sgd" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 243.0 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785240290.899925 + }, + { + "arguments": { + "optimizers": [ + "sf-sgd" + ], + "dataset": "token-bin", + "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "results_dir": "/root/autodl-tmp/llm-optimizer-results/formal", + "device": "cuda:0", + "seed": 0, + "seeds": [ + 0 + ], + "data_seed": 1337, + "stage": "tune", + "eval_tokens": [ + 16777216 + ], + "protocol_horizon_tokens": 268435456, + "lr_scales": [ + 729.0 + ], + "lr_scale_file": null, + "batch_size": 16, + "acc_steps": 8, + "sequence_length": 512, + "n_layer": 8, + "n_head": 6, + "n_embd": 384, + "warmup_steps": 16, + "eval_batches": 64, + "weight_decay": 0.1, + "grad_clip": 0.5, + "wandb": false, + "wandb_project": "llm-optimizer-sub-one-pass", + "wandb_entity": null, + "dry_run": false, + "fail_fast": false, + "rerun_completed": false + }, + "started_at_unix": 1785240368.355014 + } + ], + "runs": [ + { + "optimizer": "adamw", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "adamw", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.0003", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "adamw", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "adamw", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216", + "--lr", + "0.001", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "adamw", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "adamw", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216", + "--lr", + "0.003", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "sgd", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sgd", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.003", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "sgd", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sgd", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216", + "--lr", + "0.01", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "sgd", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sgd", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216", + "--lr", + "0.03", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "muon", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "muon", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.0003", + "--muon_lr_factor", + "0.003", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--momentum", + "0.95", + "--nesterov", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "muon", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "muon", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216", + "--lr", + "0.001", + "--muon_lr_factor", + "0.01", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--momentum", + "0.95", + "--nesterov", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "muon", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "muon", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216", + "--lr", + "0.003", + "--muon_lr_factor", + "0.03", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--momentum", + "0.95", + "--nesterov", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "soap", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "soap", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.0003", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "soap", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "soap", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216", + "--lr", + "0.001", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "soap", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "soap", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216", + "--lr", + "0.003", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "ademamix", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "ademamix", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.0003", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--adema_beta3", + "0.999", + "--adema_alpha", + "8.0", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "--adema_beta3_warmup", + "4096", + "--adema_alpha_warmup", + "4096" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "ademamix", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "ademamix", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216", + "--lr", + "0.001", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--adema_beta3", + "0.999", + "--adema_alpha", + "8.0", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "--adema_beta3_warmup", + "4096", + "--adema_alpha_warmup", + "4096" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "ademamix", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "ademamix", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216", + "--lr", + "0.003", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--adema_beta3", + "0.999", + "--adema_alpha", + "8.0", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", + "--adema_beta3_warmup", + "4096", + "--adema_alpha_warmup", + "4096" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "lion", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "lion", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.0003", + "--beta1", + "0.9", + "--beta2", + "0.99", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "lion", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "lion", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216", + "--lr", + "0.001", + "--beta1", + "0.9", + "--beta2", + "0.99", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "lion", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "lion", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216", + "--lr", + "0.003", + "--beta1", + "0.9", + "--beta2", + "0.99", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "sf-adamw", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sf-adamw", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.0003", + "--beta1", + "0.9", + "--beta2", + "0.9999", + "--scheduler", + "none", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "sf-adamw", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sf-adamw", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216", + "--lr", + "0.001", + "--beta1", + "0.9", + "--beta2", + "0.9999", + "--scheduler", + "none", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "sf-adamw", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sf-adamw", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216", + "--lr", + "0.003", + "--beta1", + "0.9", + "--beta2", + "0.9999", + "--scheduler", + "none", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "sf-sgd", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sf-sgd", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.003", + "--momentum", + "0.9", + "--scheduler", + "none", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "sf-sgd", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sf-sgd", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216", + "--lr", + "0.01", + "--momentum", + "0.9", + "--scheduler", + "none", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "sf-sgd", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sf-sgd", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216", + "--lr", + "0.03", + "--momentum", + "0.9", + "--scheduler", + "none", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "signsgd", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "signsgd", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.0003", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "signsgd", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "signsgd", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216", + "--lr", + "0.001", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "signsgd", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "signsgd", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216", + "--lr", + "0.003", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "signum", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "signum", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.0003", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "signum", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "signum", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216", + "--lr", + "0.001", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "signum", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "signum", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216", + "--lr", + "0.003", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "prodigy", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "prodigy", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.3", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--prodigy_use_bias_correction", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "prodigy", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "prodigy", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216", + "--lr", + "1", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--prodigy_use_bias_correction", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "prodigy", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "prodigy", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216", + "--lr", + "3", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--prodigy_use_bias_correction", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "sophiag", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sophiag", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.0003", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "sophiag", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sophiag", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216", + "--lr", + "0.001", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "sophiag", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sophiag", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216", + "--lr", + "0.003", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "adopt", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "adopt", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.0003", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "adopt", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "adopt", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216", + "--lr", + "0.001", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "adopt", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "adopt", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216", + "--lr", + "0.003", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "mars", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "mars", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.0003", + "--mars_lr", + "0.0009", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--mars_beta1", + "0.95", + "--mars_beta2", + "0.99", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "mars", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "mars", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216", + "--lr", + "0.001", + "--mars_lr", + "0.003", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--mars_beta1", + "0.95", + "--mars_beta2", + "0.99", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "mars", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "mars", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216", + "--lr", + "0.003", + "--mars_lr", + "0.009", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--mars_beta1", + "0.95", + "--mars_beta2", + "0.99", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "adafactor", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "adafactor", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.0003", + "--beta1", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "adafactor", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "adafactor", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216", + "--lr", + "0.001", + "--beta1", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "adafactor", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "adafactor", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216", + "--lr", + "0.003", + "--beta1", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "lamb", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "lamb", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.0003", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "lamb", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "lamb", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216", + "--lr", + "0.001", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "lamb", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "lamb", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216", + "--lr", + "0.003", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "scion", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "scion", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.0003", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "scion", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "scion", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216", + "--lr", + "0.001", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "scion", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "scion", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216", + "--lr", + "0.003", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "scion-light", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "scion-light", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.0003", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "scion-light", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "scion-light", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216", + "--lr", + "0.001", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "scion-light", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "scion-light", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216", + "--lr", + "0.003", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "d-muon", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/torchrun", + "--standalone", + "--nproc_per_node=1", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--distributed_backend", + "nccl", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "d-muon", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.0003", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--momentum", + "0.95", + "--nesterov", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "d-muon", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/torchrun", + "--standalone", + "--nproc_per_node=1", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--distributed_backend", + "nccl", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "d-muon", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216", + "--lr", + "0.001", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--momentum", + "0.95", + "--nesterov", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "d-muon", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/torchrun", + "--standalone", + "--nproc_per_node=1", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--distributed_backend", + "nccl", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "d-muon", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216", + "--lr", + "0.003", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--momentum", + "0.95", + "--nesterov", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "muon-pytorch", + "seed": 0, + "lr_scale": 0.3, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "muon-pytorch", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216", + "--lr", + "0.006", + "--muon_adamw_lr", + "0.0003", + "--momentum", + "0.95", + "--nesterov", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "muon-pytorch", + "seed": 0, + "lr_scale": 1.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "muon-pytorch", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216", + "--lr", + "0.02", + "--muon_adamw_lr", + "0.001", + "--momentum", + "0.95", + "--nesterov", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "muon-pytorch", + "seed": 0, + "lr_scale": 3.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "muon-pytorch", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216", + "--lr", + "0.06", + "--muon_adamw_lr", + "0.003", + "--momentum", + "0.95", + "--nesterov", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216", + "status": "skipped-completed" + }, + { + "optimizer": "adafactor", + "seed": 0, + "lr_scale": 0.09999999999999999, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "adafactor", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216", + "--lr", + "0.0001", + "--beta1", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216", + "status": "completed", + "started_at_unix": 1785236958.206393, + "returncode": 0, + "finished_at_unix": 1785237039.866649 + }, + { + "optimizer": "adopt", + "seed": 0, + "lr_scale": 0.09999999999999999, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "adopt", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216", + "--lr", + "0.0001", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216", + "status": "completed", + "started_at_unix": 1785237039.9119153, + "returncode": 0, + "finished_at_unix": 1785237118.1727488 + }, + { + "optimizer": "d-muon", + "seed": 0, + "lr_scale": 9.0, + "command": [ + "/root/miniconda3/bin/torchrun", + "--standalone", + "--nproc_per_node=1", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--distributed_backend", + "nccl", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "d-muon", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216", + "--lr", + "0.009", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--momentum", + "0.95", + "--nesterov", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216", + "status": "completed", + "started_at_unix": 1785237118.217154, + "returncode": 0, + "finished_at_unix": 1785237200.1304214 + }, + { + "optimizer": "lamb", + "seed": 0, + "lr_scale": 9.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "lamb", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216", + "--lr", + "0.009", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216", + "status": "completed", + "started_at_unix": 1785237200.1762109, + "returncode": 0, + "finished_at_unix": 1785237280.2982078 + }, + { + "optimizer": "lion", + "seed": 0, + "lr_scale": 0.09999999999999999, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "lion", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216", + "--lr", + "0.0001", + "--beta1", + "0.9", + "--beta2", + "0.99", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216", + "status": "completed", + "started_at_unix": 1785237280.3438804, + "returncode": 0, + "finished_at_unix": 1785237359.6563318 + }, + { + "optimizer": "mars", + "seed": 0, + "lr_scale": 0.09999999999999999, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "mars", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216", + "--lr", + "0.0001", + "--mars_lr", + "0.0003", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--mars_beta1", + "0.95", + "--mars_beta2", + "0.99", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216", + "status": "completed", + "started_at_unix": 1785237359.7018409, + "returncode": 0, + "finished_at_unix": 1785237439.8954363 + }, + { + "optimizer": "muon", + "seed": 0, + "lr_scale": 9.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "muon", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216", + "--lr", + "0.009", + "--muon_lr_factor", + "0.09", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--momentum", + "0.95", + "--nesterov", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216", + "status": "completed", + "started_at_unix": 1785237439.9406068, + "returncode": 0, + "finished_at_unix": 1785237520.877728 + }, + { + "optimizer": "muon-pytorch", + "seed": 0, + "lr_scale": 9.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "muon-pytorch", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216", + "--lr", + "0.18", + "--muon_adamw_lr", + "0.009", + "--momentum", + "0.95", + "--nesterov", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216", + "status": "completed", + "started_at_unix": 1785237520.923198, + "returncode": 0, + "finished_at_unix": 1785237600.6159189 + }, + { + "optimizer": "prodigy", + "seed": 0, + "lr_scale": 0.09999999999999999, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "prodigy", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216", + "--lr", + "0.1", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--prodigy_use_bias_correction", + "True", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216", + "status": "completed", + "started_at_unix": 1785237600.6616597, + "returncode": 0, + "finished_at_unix": 1785237681.0280638 + }, + { + "optimizer": "scion", + "seed": 0, + "lr_scale": 0.09999999999999999, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "scion", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216", + "--lr", + "0.0001", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216", + "status": "completed", + "started_at_unix": 1785237681.0745208, + "returncode": 0, + "finished_at_unix": 1785237762.353516 + }, + { + "optimizer": "scion-light", + "seed": 0, + "lr_scale": 0.09999999999999999, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "scion-light", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216", + "--lr", + "0.0001", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216", + "status": "completed", + "started_at_unix": 1785237762.4001708, + "returncode": 0, + "finished_at_unix": 1785237843.4973056 + }, + { + "optimizer": "sf-sgd", + "seed": 0, + "lr_scale": 9.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sf-sgd", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216", + "--lr", + "0.09", + "--momentum", + "0.9", + "--scheduler", + "none", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216", + "status": "completed", + "started_at_unix": 1785237843.5444808, + "returncode": 0, + "finished_at_unix": 1785237921.6364317 + }, + { + "optimizer": "sgd", + "seed": 0, + "lr_scale": 9.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sgd", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216", + "--lr", + "0.09", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216", + "status": "completed", + "started_at_unix": 1785237921.6823702, + "returncode": 0, + "finished_at_unix": 1785237999.9294279 + }, + { + "optimizer": "signsgd", + "seed": 0, + "lr_scale": 0.09999999999999999, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "signsgd", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216", + "--lr", + "0.0001", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216", + "status": "completed", + "started_at_unix": 1785237999.9738657, + "returncode": 0, + "finished_at_unix": 1785238078.8351915 + }, + { + "optimizer": "sophiag", + "seed": 0, + "lr_scale": 0.09999999999999999, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sophiag", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216", + "--lr", + "0.0001", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216", + "status": "completed", + "started_at_unix": 1785238078.8798544, + "returncode": 0, + "finished_at_unix": 1785238159.400037 + }, + { + "optimizer": "lamb", + "seed": 0, + "lr_scale": 27.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "lamb", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216", + "--lr", + "0.027", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216", + "status": "completed", + "started_at_unix": 1785238159.483625, + "returncode": 0, + "finished_at_unix": 1785238240.1272843 + }, + { + "optimizer": "sf-sgd", + "seed": 0, + "lr_scale": 27.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sf-sgd", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216", + "--lr", + "0.27", + "--momentum", + "0.9", + "--scheduler", + "none", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216", + "status": "completed", + "started_at_unix": 1785238240.1752775, + "returncode": 0, + "finished_at_unix": 1785238318.2678187 + }, + { + "optimizer": "sgd", + "seed": 0, + "lr_scale": 27.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sgd", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216", + "--lr", + "0.27", + "--momentum", + "0.9", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216", + "status": "completed", + "started_at_unix": 1785238318.3131683, + "returncode": 0, + "finished_at_unix": 1785238396.6095943 + }, + { + "optimizer": "lamb", + "seed": 0, + "lr_scale": 81.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "lamb", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216", + "--lr", + "0.081", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216", + "status": "completed", + "started_at_unix": 1785240132.4824207, + "returncode": 0, + "finished_at_unix": 1785240213.2773643 + }, + { + "optimizer": "sf-sgd", + "seed": 0, + "lr_scale": 81.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sf-sgd", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216", + "--lr", + "0.81", + "--momentum", + "0.9", + "--scheduler", + "none", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216", + "status": "completed", + "started_at_unix": 1785240213.3222752, + "returncode": 0, + "finished_at_unix": 1785240290.821528 + }, + { + "optimizer": "sf-sgd", + "seed": 0, + "lr_scale": 243.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sf-sgd", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216", + "--lr", + "2.43", + "--momentum", + "0.9", + "--scheduler", + "none", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216", + "status": "completed", + "started_at_unix": 1785240290.9041584, + "returncode": 0, + "finished_at_unix": 1785240368.2747993 + }, + { + "optimizer": "sf-sgd", + "seed": 0, + "lr_scale": 729.0, + "command": [ + "/root/miniconda3/bin/python3.12", + "/root/autodl-tmp/llm-optimizer-benchmark/src/main.py", + "--config_format", + "base", + "--model", + "llama", + "--dataset", + "token-bin", + "--datasets_dir", + "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", + "--device", + "cuda:0", + "--n_layer", + "8", + "--n_head", + "6", + "--n_embd", + "384", + "--batch_size", + "16", + "--acc_steps", + "8", + "--sequence_length", + "512", + "--opt", + "sf-sgd", + "--train_token_budget", + "16777216", + "--eval_at_tokens", + "16777216", + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + "16", + "--eval_interval", + "0", + "--eval_batches", + "64", + "--limit_final_eval", + "--latest_ckpt_interval", + "256", + "--log_interval", + "50", + "--weight_decay", + "0.1", + "--grad_clip", + "0.5", + "--seed", + "0", + "--data_seed", + "1337", + "--results_base_folder", + "/root/autodl-tmp/llm-optimizer-results/formal", + "--experiment_name", + "sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216", + "--lr", + "7.29", + "--momentum", + "0.9", + "--scheduler", + "none", + "--train_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", + "--val_data_path", + "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin" + ], + "experiment_dir": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216", + "status": "completed", + "started_at_unix": 1785240368.3594096, + "returncode": 0, + "finished_at_unix": 1785240445.655429 + } + ] +} diff --git a/exps/sub_one_pass/formal/pipeline_state.json b/exps/sub_one_pass/formal/pipeline_state.json new file mode 100644 index 0000000..b2bb7d7 --- /dev/null +++ b/exps/sub_one_pass/formal/pipeline_state.json @@ -0,0 +1,23 @@ +{ + "stage": "completed", + "updated_at_unix": 1785273368.298487, + "s3_completed": true, + "selected_optimizers": [ + "adamw", + "ademamix", + "soap", + "muon" + ], + "seeds": [ + 1, + 2 + ], + "main_manifest_counts": { + "completed": 20 + }, + "confirm_manifest_counts": { + "completed": 8 + }, + "selection_file": "/root/autodl-tmp/llm-optimizer-results/formal/confirm_selection.json", + "summary_csv": "/root/autodl-tmp/llm-optimizer-results/formal/sub_one_pass_results.csv" +} diff --git a/exps/sub_one_pass/formal/selected_lr_scales.json b/exps/sub_one_pass/formal/selected_lr_scales.json new file mode 100644 index 0000000..ab32540 --- /dev/null +++ b/exps/sub_one_pass/formal/selected_lr_scales.json @@ -0,0 +1,22 @@ +{ + "adafactor": 0.3, + "adamw": 1.0, + "ademamix": 1.0, + "adopt": 0.3, + "d-muon": 3.0, + "lamb": 27.0, + "lion": 0.3, + "mars": 0.3, + "muon": 3.0, + "muon-pytorch": 3.0, + "prodigy": 0.3, + "scion": 0.3, + "scion-light": 0.3, + "sf-adamw": 1.0, + "sf-sgd": 729.0, + "sgd": 9.0, + "signsgd": 0.3, + "signum": 1.0, + "soap": 1.0, + "sophiag": 0.3 +} diff --git a/exps/sub_one_pass/formal/selected_lr_scales_report.json b/exps/sub_one_pass/formal/selected_lr_scales_report.json new file mode 100644 index 0000000..05af1fc --- /dev/null +++ b/exps/sub_one_pass/formal/selected_lr_scales_report.json @@ -0,0 +1,552 @@ +{ + "adafactor": { + "candidates": [ + { + "lr_scale": 0.09999999999999999, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216/summary.json", + "val_loss": 5.9108781814575195 + }, + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 5.906637191772461 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 5.910003662109375 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 5.909377574920654 + } + ], + "selected_lr_scale": 0.3, + "selected_val_loss": 5.906637191772461, + "selection_on_grid_boundary": false + }, + "adamw": { + "candidates": [ + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 5.825977802276611 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 5.633615016937256 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 6.316389083862305 + } + ], + "selected_lr_scale": 1.0, + "selected_val_loss": 5.633615016937256, + "selection_on_grid_boundary": false + }, + "ademamix": { + "candidates": [ + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 5.787484645843506 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 5.664802074432373 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 6.46177864074707 + } + ], + "selected_lr_scale": 1.0, + "selected_val_loss": 5.664802074432373, + "selection_on_grid_boundary": false + }, + "adopt": { + "candidates": [ + { + "lr_scale": 0.09999999999999999, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216/summary.json", + "val_loss": 6.21639347076416 + }, + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 5.876038551330566 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 6.103500843048096 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 6.369908332824707 + } + ], + "selected_lr_scale": 0.3, + "selected_val_loss": 5.876038551330566, + "selection_on_grid_boundary": false + }, + "d-muon": { + "candidates": [ + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 6.030533790588379 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 5.397306442260742 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 5.1279730796813965 + }, + { + "lr_scale": 9.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216/summary.json", + "val_loss": 5.191718101501465 + } + ], + "selected_lr_scale": 3.0, + "selected_val_loss": 5.1279730796813965, + "selection_on_grid_boundary": false + }, + "lamb": { + "candidates": [ + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 9.87247085571289 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 9.244894027709961 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 7.944293975830078 + }, + { + "lr_scale": 9.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216/summary.json", + "val_loss": 6.3077802658081055 + }, + { + "lr_scale": 27.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216/summary.json", + "val_loss": 5.966597557067871 + }, + { + "lr_scale": 81.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216/summary.json", + "val_loss": 6.549368381500244 + } + ], + "selected_lr_scale": 27.0, + "selected_val_loss": 5.966597557067871, + "selection_on_grid_boundary": false + }, + "lion": { + "candidates": [ + { + "lr_scale": 0.09999999999999999, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216/summary.json", + "val_loss": 6.423236846923828 + }, + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 6.276636123657227 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 6.849046230316162 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 7.386219024658203 + } + ], + "selected_lr_scale": 0.3, + "selected_val_loss": 6.276636123657227, + "selection_on_grid_boundary": false + }, + "mars": { + "candidates": [ + { + "lr_scale": 0.09999999999999999, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216/summary.json", + "val_loss": 6.105465412139893 + }, + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 5.844259262084961 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 6.1865925788879395 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 6.590240478515625 + } + ], + "selected_lr_scale": 0.3, + "selected_val_loss": 5.844259262084961, + "selection_on_grid_boundary": false + }, + "muon": { + "candidates": [ + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 5.847861289978027 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 5.261423110961914 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 5.157479286193848 + }, + { + "lr_scale": 9.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216/summary.json", + "val_loss": 5.522243499755859 + } + ], + "selected_lr_scale": 3.0, + "selected_val_loss": 5.157479286193848, + "selection_on_grid_boundary": false + }, + "muon-pytorch": { + "candidates": [ + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 5.793531894683838 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 5.356301784515381 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 5.283370018005371 + }, + { + "lr_scale": 9.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216/summary.json", + "val_loss": 5.898062705993652 + } + ], + "selected_lr_scale": 3.0, + "selected_val_loss": 5.283370018005371, + "selection_on_grid_boundary": false + }, + "prodigy": { + "candidates": [ + { + "lr_scale": 0.09999999999999999, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216/summary.json", + "val_loss": 6.069522857666016 + }, + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 5.899776458740234 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 6.313230991363525 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 6.386815071105957 + } + ], + "selected_lr_scale": 0.3, + "selected_val_loss": 5.899776458740234, + "selection_on_grid_boundary": false + }, + "scion": { + "candidates": [ + { + "lr_scale": 0.09999999999999999, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216/summary.json", + "val_loss": 6.358173370361328 + }, + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 6.25476598739624 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 6.319926738739014 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 6.581727027893066 + } + ], + "selected_lr_scale": 0.3, + "selected_val_loss": 6.25476598739624, + "selection_on_grid_boundary": false + }, + "scion-light": { + "candidates": [ + { + "lr_scale": 0.09999999999999999, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216/summary.json", + "val_loss": 6.558917999267578 + }, + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 6.453986167907715 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 6.479294776916504 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 6.810400009155273 + } + ], + "selected_lr_scale": 0.3, + "selected_val_loss": 6.453986167907715, + "selection_on_grid_boundary": false + }, + "sf-adamw": { + "candidates": [ + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 6.211974143981934 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 6.035920143127441 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 6.4182233810424805 + } + ], + "selected_lr_scale": 1.0, + "selected_val_loss": 6.035920143127441, + "selection_on_grid_boundary": false + }, + "sf-sgd": { + "candidates": [ + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 10.325323104858398 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 9.711913108825684 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 8.906181335449219 + }, + { + "lr_scale": 9.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216/summary.json", + "val_loss": 8.234554290771484 + }, + { + "lr_scale": 27.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216/summary.json", + "val_loss": 7.711722373962402 + }, + { + "lr_scale": 81.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216/summary.json", + "val_loss": 7.60025691986084 + }, + { + "lr_scale": 243.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216/summary.json", + "val_loss": 7.5896501541137695 + }, + { + "lr_scale": 729.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216/summary.json", + "val_loss": 7.577207565307617 + } + ], + "selected_lr_scale": 729.0, + "selected_val_loss": 7.577207565307617, + "selection_on_grid_boundary": true + }, + "sgd": { + "candidates": [ + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 8.42439079284668 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 7.8472442626953125 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 7.7101545333862305 + }, + { + "lr_scale": 9.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216/summary.json", + "val_loss": 7.59018611907959 + }, + { + "lr_scale": 27.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216/summary.json", + "val_loss": 7.623948574066162 + } + ], + "selected_lr_scale": 9.0, + "selected_val_loss": 7.59018611907959, + "selection_on_grid_boundary": false + }, + "signsgd": { + "candidates": [ + { + "lr_scale": 0.09999999999999999, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216/summary.json", + "val_loss": 6.661827087402344 + }, + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 6.475642204284668 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 6.683113098144531 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 6.854705810546875 + } + ], + "selected_lr_scale": 0.3, + "selected_val_loss": 6.475642204284668, + "selection_on_grid_boundary": false + }, + "signum": { + "candidates": [ + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 6.296080112457275 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 6.2198805809021 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 6.384660720825195 + } + ], + "selected_lr_scale": 1.0, + "selected_val_loss": 6.2198805809021, + "selection_on_grid_boundary": false + }, + "soap": { + "candidates": [ + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 6.225244998931885 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 5.291918754577637 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 5.5883636474609375 + } + ], + "selected_lr_scale": 1.0, + "selected_val_loss": 5.291918754577637, + "selection_on_grid_boundary": false + }, + "sophiag": { + "candidates": [ + { + "lr_scale": 0.09999999999999999, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216/summary.json", + "val_loss": 6.111329078674316 + }, + { + "lr_scale": 0.3, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216/summary.json", + "val_loss": 6.050726890563965 + }, + { + "lr_scale": 1.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216/summary.json", + "val_loss": 6.201114654541016 + }, + { + "lr_scale": 3.0, + "summary": "/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216/summary.json", + "val_loss": 6.950982093811035 + } + ], + "selected_lr_scale": 0.3, + "selected_val_loss": 6.050726890563965, + "selection_on_grid_boundary": false + } +} diff --git a/exps/sub_one_pass/formal/sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456/run.log new file mode 100644 index 0000000..e85e316 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456/run.log @@ -0,0 +1,156 @@ +Starting Experiment: sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456', 'seed': 1, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +using fused AdamW: True + +Optimizer: +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.001 + maximize: False + weight_decay: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.001 + maximize: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.384 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.392 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.037 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.908 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.782 iter_dt=2.70e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.624 val_pp=277.027 val_acc=0.185951 +Train: Iter=300 (0.036622 effective passes) train_loss=5.412 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.606 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=5.267 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=5.065 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=5.028 iter_dt=2.71e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.092 val_pp=162.725 val_acc=0.219788 +Train: Iter=550 (0.067140 effective passes) train_loss=5.155 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.707 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=5.098 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.916 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.899 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.710 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.659 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.589 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.638 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.638 iter_dt=2.71e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.568 val_pp=96.390 val_acc=0.268028 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.684 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.489 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.505 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.432 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.448 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.498 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.518 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.519 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.373 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.459 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.375 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.641 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.210 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.557 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.241 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.261 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.564 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.166 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.115 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.297 iter_dt=2.71e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.287 val_pp=72.727 val_acc=0.291880 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.297 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.315 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.268 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.261 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.296 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.007 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.201 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.204 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.197 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.448 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.247 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.012 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.437 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.325 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.216 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.052 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.368 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.272 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.217 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.080 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.246 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.876 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.111 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.185 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.260 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.120 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.326 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.040 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.022 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.111 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.183 iter_dt=2.62e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.255 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.189 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.053 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.072 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.157 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.995 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.087 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.173 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.198 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.096 iter_dt=2.70e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.098 val_pp=60.219 val_acc=0.307306 diff --git a/exps/sub_one_pass/formal/sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456/summary.json new file mode 100644 index 0000000..aaa93d1 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.3836259841918945, 6.392024993896484, 6.036675453186035, 5.908082962036133, 5.781793117523193, 5.411533355712891, 5.606035232543945, 5.267053127288818, 5.064592361450195, 5.0282368659973145, 5.15476131439209, 4.706686019897461, 5.0975141525268555, 4.91586971282959, 4.898597240447998, 4.709891319274902, 4.658722877502441, 4.588845252990723, 4.637593746185303, 4.6382060050964355, 4.68353796005249, 4.489434242248535, 4.505237579345703, 4.432445526123047, 4.448126792907715, 4.497901439666748, 4.518367767333984, 4.519224166870117, 4.373392105102539, 4.458596229553223, 4.375406742095947, 4.6405110359191895, 4.210110664367676, 4.556856632232666, 4.241228103637695, 4.260550498962402, 4.564141273498535, 4.166138648986816, 4.115278244018555, 4.297308444976807, 4.296852111816406, 4.314964294433594, 4.267871379852295, 4.26090669631958, 4.295799255371094, 4.0070905685424805, 4.20144510269165, 4.203568935394287, 4.197328090667725, 4.44770622253418, 4.247437953948975, 4.012086391448975, 4.436972618103027, 4.325494289398193, 4.216436386108398, 4.052199363708496, 4.368090629577637, 4.272452354431152, 4.2166948318481445, 4.079593658447266, 4.246006011962891, 3.8759284019470215, 4.111288070678711, 4.184513092041016, 4.260124206542969, 4.119894504547119, 4.325753688812256, 4.039510726928711, 4.021529197692871, 4.1107177734375, 4.183474540710449, 4.254629135131836, 4.1887102127075195, 4.052520751953125, 4.0720343589782715, 4.157426357269287, 3.99473237991333, 4.0874481201171875, 4.172501564025879, 4.198118209838867, 4.095864772796631], "val_loss": [5.624117851257324, 5.092065811157227, 4.568408966064453, 4.286713123321533, 4.097991943359375], "val_pp": [277.02674896318183, 162.72511822087296, 96.39033074510401, 72.72682072742225, 60.2190764599356], "val_acc": [0.18595123291015625, 0.21978759765625, 0.26802825927734375, 0.29187965393066406, 0.30730628967285156], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.624117851257324, "val_perplexity": 277.02674896318183, "val_accuracy": 0.18595123291015625, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 5.092065811157227, "val_perplexity": 162.72511822087296, "val_accuracy": 0.21978759765625, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.568408966064453, "val_perplexity": 96.39033074510401, "val_accuracy": 0.26802825927734375, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.286713123321533, "val_perplexity": 72.72682072742225, "val_accuracy": 0.29187965393066406, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.097991943359375, "val_perplexity": 60.2190764599356, "val_accuracy": 0.30730628967285156, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.274113945, 0.269073144, 0.271640815, 0.270322831, 0.270410331, 0.271803652, 0.270994925, 0.272623288, 0.270762579, 0.270544665, 0.270233966, 0.270621826, 0.270543909, 0.271683289, 0.271518091, 0.276858079, 0.270866135, 0.270523841, 0.272059361, 0.2706417, 0.269672982, 0.269326581, 0.270792262, 0.27171245, 0.271053667, 0.27241212, 0.271882769, 0.270307911, 0.270190163, 0.27010703, 0.270820662, 0.270932694, 0.27026715, 0.275803166, 0.271478999, 0.272303311, 0.270382713, 0.270127069, 0.272300781, 0.271409527, 0.271571641, 0.270175569, 0.271824751, 0.271542172, 0.270268293, 0.27119179, 0.271309819, 0.272111265, 0.269933292, 0.270985745, 0.270792304, 0.271207006, 0.271341102, 0.271915937, 0.275313577, 0.272230806, 0.2725438, 0.272650766, 0.272308555, 0.271905517, 0.273482363, 0.271256421, 0.272481401, 0.271945993, 0.271175212, 0.2713346, 0.27057304, 0.272547139, 0.275308281, 0.271038285, 0.261673062, 0.270367856, 0.270169486, 0.270354727, 0.271162409, 0.270564781, 0.273862301, 0.271114733, 0.26988889, 0.270432087, 0.269975645], "train_time_seconds": 1110.7638108969995, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 241667.4484409286, "peak_memory_bytes": 6175430656, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456", "seed": 1, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "adamw", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456/run.log new file mode 100644 index 0000000..7ebfafe --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456/run.log @@ -0,0 +1,156 @@ +Starting Experiment: sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456', 'seed': 2, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +using fused AdamW: True + +Optimizer: +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.001 + maximize: False + weight_decay: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.001 + maximize: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.388 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.476 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.153 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.078 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.895 iter_dt=2.73e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.800 val_pp=330.217 val_acc=0.175360 +Train: Iter=300 (0.036622 effective passes) train_loss=5.530 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.678 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=5.407 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=5.160 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=5.063 iter_dt=2.79e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.138 val_pp=170.332 val_acc=0.215649 +Train: Iter=550 (0.067140 effective passes) train_loss=5.192 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.750 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=5.113 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.949 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.926 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.754 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.681 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.612 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.668 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.674 iter_dt=2.79e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.594 val_pp=98.895 val_acc=0.267216 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.725 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.511 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.518 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.450 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.485 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.527 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.521 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.542 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.396 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.474 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.396 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.678 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.224 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.583 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.260 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.291 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.594 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.190 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.146 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.335 iter_dt=2.71e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.306 val_pp=74.144 val_acc=0.290478 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.314 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.333 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.284 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.266 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.319 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.035 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.215 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.218 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.199 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.476 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.248 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.044 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.453 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.330 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.223 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.075 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.375 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.296 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.233 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.076 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.254 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.886 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.132 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.192 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.264 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.137 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.329 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.059 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.029 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.121 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.199 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.265 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.210 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.074 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.079 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.192 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.009 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.107 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.181 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.209 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.093 iter_dt=2.76e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.109 val_pp=60.879 val_acc=0.306969 diff --git a/exps/sub_one_pass/formal/sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456/summary.json new file mode 100644 index 0000000..faceee7 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.3883748054504395, 6.47587776184082, 6.153181076049805, 6.077906608581543, 5.895094394683838, 5.529954433441162, 5.677595615386963, 5.407329082489014, 5.159956932067871, 5.063474655151367, 5.191830635070801, 4.749785423278809, 5.113203048706055, 4.949028015136719, 4.926328659057617, 4.753521919250488, 4.681099891662598, 4.611785888671875, 4.6679840087890625, 4.673863410949707, 4.724526882171631, 4.510897636413574, 4.518113613128662, 4.450248718261719, 4.484837532043457, 4.526832580566406, 4.5207929611206055, 4.541802406311035, 4.396439075469971, 4.474046230316162, 4.39553165435791, 4.677821636199951, 4.223964691162109, 4.58344841003418, 4.260140419006348, 4.290529727935791, 4.59360933303833, 4.189587593078613, 4.1458563804626465, 4.334957122802734, 4.314265251159668, 4.33303689956665, 4.2838544845581055, 4.26644229888916, 4.318719863891602, 4.0348310470581055, 4.214994430541992, 4.2180562019348145, 4.1990227699279785, 4.475931167602539, 4.247845649719238, 4.044405937194824, 4.45261287689209, 4.330235958099365, 4.222575664520264, 4.075463771820068, 4.375452995300293, 4.296191215515137, 4.232864856719971, 4.076229095458984, 4.254164218902588, 3.8859832286834717, 4.131711006164551, 4.192107200622559, 4.264070510864258, 4.1368937492370605, 4.328533172607422, 4.059211254119873, 4.029256820678711, 4.121467590332031, 4.199174880981445, 4.264911651611328, 4.20980167388916, 4.073877334594727, 4.0787482261657715, 4.192357063293457, 4.009357452392578, 4.106991767883301, 4.181373596191406, 4.208764553070068, 4.092961311340332], "val_loss": [5.799754619598389, 5.137753486633301, 4.594059467315674, 4.306008338928223, 4.108890533447266], "val_pp": [330.21723256127666, 170.33209487378295, 98.89477213389124, 74.14372523990149, 60.87896846029916], "val_acc": [0.17535972595214844, 0.21564865112304688, 0.2672157287597656, 0.2904777526855469, 0.30696868896484375], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.799754619598389, "val_perplexity": 330.21723256127666, "val_accuracy": 0.17535972595214844, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 5.137753486633301, "val_perplexity": 170.33209487378295, "val_accuracy": 0.21564865112304688, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.594059467315674, "val_perplexity": 98.89477213389124, "val_accuracy": 0.2672157287597656, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.306008338928223, "val_perplexity": 74.14372523990149, "val_accuracy": 0.2904777526855469, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.108890533447266, "val_perplexity": 60.87896846029916, "val_accuracy": 0.30696868896484375, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.271710904, 0.270779095, 0.273829174, 0.274953236, 0.273069869, 0.272684201, 0.278451053, 0.273706934, 0.277539984, 0.278951248, 0.27911079, 0.27810958, 0.275879919, 0.276939216, 0.277734451, 0.276381925, 0.281934289, 0.281131531, 0.280061251, 0.278933307, 0.272805602, 0.273232526, 0.271440903, 0.274360855, 0.274769794, 0.272980791, 0.275738561, 0.271669276, 0.270539238, 0.270768202, 0.272548499, 0.272473451, 0.269683445, 0.271385268, 0.269111581, 0.274211535, 0.272199433, 0.278259979, 0.27143253, 0.271333003, 0.271049643, 0.27168061, 0.271510029, 0.271026572, 0.271131146, 0.272910198, 0.271668415, 0.271425538, 0.271547348, 0.270690498, 0.271200616, 0.271822277, 0.27009008, 0.270238284, 0.270169686, 0.270979028, 0.271426383, 0.269768063, 0.271945819, 0.272138819, 0.270615313, 0.271313415, 0.270583465, 0.270256565, 0.272787977, 0.26978207, 0.272080504, 0.269504491, 0.269932392, 0.270845417, 0.271260318, 0.271461215, 0.277901552, 0.270957304, 0.271926349, 0.270635286, 0.269746225, 0.270318948, 0.27615043, 0.274001953, 0.27635487], "train_time_seconds": 1117.7748976870016, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 240151.62315370503, "peak_memory_bytes": 6175430656, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456", "seed": 2, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "adamw", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456/run.log new file mode 100644 index 0000000..73cc723 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456/run.log @@ -0,0 +1,147 @@ +Starting Experiment: sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456', 'seed': 1, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'ademamix', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.999, 'adema_alpha': 8.0, 'adema_beta3_warmup': 4096, 'adema_alpha_warmup': 4096, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdEMAMix ( +Parameter Group 0 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.001 + weight_decay: 0.1 + +Parameter Group 1 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.001 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.334 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.454 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.111 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.959 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.798 iter_dt=2.76e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.653 val_pp=285.078 val_acc=0.183714 +Train: Iter=300 (0.036622 effective passes) train_loss=5.404 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.587 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=5.259 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=5.030 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=4.982 iter_dt=2.73e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.039 val_pp=154.268 val_acc=0.223021 +Train: Iter=550 (0.067140 effective passes) train_loss=5.075 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.659 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=5.030 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.849 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.849 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.643 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.568 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.520 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.587 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.597 iter_dt=2.81e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.499 val_pp=89.909 val_acc=0.274397 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.621 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.416 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.441 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.362 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.382 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.419 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.423 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.449 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.301 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.360 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.307 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.571 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.133 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.497 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.167 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.190 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.503 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.099 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.046 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.235 iter_dt=2.75e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.205 val_pp=67.009 val_acc=0.300161 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.212 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.233 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.182 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.177 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.233 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.940 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.125 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.114 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.126 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.378 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.151 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.938 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.365 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.240 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.126 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.985 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.283 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.199 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.148 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=3.985 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.162 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.793 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.049 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.115 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.175 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.025 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.238 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=3.973 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.935 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.030 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.098 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.180 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.106 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=3.995 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=3.973 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.079 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.919 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.017 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.071 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.110 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.019 iter_dt=2.73e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.020 val_pp=55.683 val_acc=0.316698 diff --git a/exps/sub_one_pass/formal/sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456/summary.json new file mode 100644 index 0000000..124ccf7 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.334347724914551, 6.453817367553711, 6.111137390136719, 5.959168910980225, 5.797872543334961, 5.404384136199951, 5.5873565673828125, 5.2591118812561035, 5.030117511749268, 4.982424736022949, 5.074672698974609, 4.659334182739258, 5.0300493240356445, 4.849174499511719, 4.848738670349121, 4.643414497375488, 4.5680718421936035, 4.520395278930664, 4.5866379737854, 4.596683502197266, 4.620975494384766, 4.416282653808594, 4.44062614440918, 4.362144470214844, 4.381796836853027, 4.418586730957031, 4.422685623168945, 4.449494361877441, 4.300611972808838, 4.359533309936523, 4.306910514831543, 4.571313858032227, 4.133028984069824, 4.49662971496582, 4.166658401489258, 4.1897382736206055, 4.503109455108643, 4.0989227294921875, 4.0461578369140625, 4.234601974487305, 4.2121968269348145, 4.233315467834473, 4.182136535644531, 4.177183151245117, 4.232937812805176, 3.9404478073120117, 4.124853610992432, 4.1136794090271, 4.126416206359863, 4.378244400024414, 4.150742053985596, 3.938015937805176, 4.365474224090576, 4.239525318145752, 4.1259074211120605, 3.98539400100708, 4.283106803894043, 4.198538780212402, 4.1481242179870605, 3.9847488403320312, 4.162318229675293, 3.792741060256958, 4.048703193664551, 4.115447044372559, 4.175170421600342, 4.025485992431641, 4.237751007080078, 3.972834587097168, 3.935195207595825, 4.0295796394348145, 4.097561836242676, 4.17965030670166, 4.106426239013672, 3.9953975677490234, 3.9729480743408203, 4.0794243812561035, 3.9191837310791016, 4.017434120178223, 4.07057523727417, 4.109838008880615, 4.019491195678711], "val_loss": [5.652766227722168, 5.038692951202393, 4.498805046081543, 4.204828262329102, 4.019683837890625], "val_pp": [285.0778854953414, 154.26772420847368, 89.90935714143036, 67.00889916573766, 55.68334747159574], "val_acc": [0.1837139129638672, 0.2230205535888672, 0.2743968963623047, 0.30016136169433594, 0.3166980743408203], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.652766227722168, "val_perplexity": 285.0778854953414, "val_accuracy": 0.1837139129638672, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 5.038692951202393, "val_perplexity": 154.26772420847368, "val_accuracy": 0.2230205535888672, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.498805046081543, "val_perplexity": 89.90935714143036, "val_accuracy": 0.2743968963623047, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.204828262329102, "val_perplexity": 67.00889916573766, "val_accuracy": 0.30016136169433594, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.019683837890625, "val_perplexity": 55.68334747159574, "val_accuracy": 0.3166980743408203, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.274996667, 0.275294216, 0.272426723, 0.272994445, 0.27641894, 0.277699843, 0.275405997, 0.274285199, 0.27495936, 0.273413851, 0.27462291, 0.274753641, 0.274393263, 0.276277761, 0.273530558, 0.274856568, 0.276430355, 0.277905435, 0.276657911, 0.281086531, 0.281879684, 0.276887496, 0.276437179, 0.274422544, 0.275380386, 0.27569996, 0.277532522, 0.27589715, 0.275144987, 0.276595061, 0.276578679, 0.278506929, 0.273536258, 0.277651658, 0.274453174, 0.273671924, 0.276574842, 0.276392422, 0.274883873, 0.274527866, 0.275613563, 0.273677467, 0.273849577, 0.273460539, 0.273543701, 0.274402948, 0.278539186, 0.279050086, 0.276913712, 0.275054494, 0.275543312, 0.27671046, 0.274997607, 0.275408346, 0.27514859, 0.27493737, 0.275127003, 0.274779811, 0.274873613, 0.275469319, 0.276867895, 0.276603087, 0.276367206, 0.275844148, 0.275590894, 0.274684787, 0.274813715, 0.276008143, 0.27581826, 0.27593928, 0.277381735, 0.275297265, 0.274965003, 0.276155448, 0.274022161, 0.273256782, 0.272908792, 0.273451869, 0.274109929, 0.275387639, 0.273052986], "train_time_seconds": 1128.150925535, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 237942.85846346364, "peak_memory_bytes": 6314994176, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456", "seed": 1, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "ademamix", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.999, "adema_alpha": 8.0, "adema_beta3_warmup": 4096, "adema_alpha_warmup": 4096, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456/run.log new file mode 100644 index 0000000..8bd6bf0 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456/run.log @@ -0,0 +1,147 @@ +Starting Experiment: sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456', 'seed': 2, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'ademamix', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.999, 'adema_alpha': 8.0, 'adema_beta3_warmup': 4096, 'adema_alpha_warmup': 4096, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdEMAMix ( +Parameter Group 0 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.001 + weight_decay: 0.1 + +Parameter Group 1 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.001 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.357 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.503 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.109 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.980 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.803 iter_dt=2.74e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.665 val_pp=288.511 val_acc=0.182850 +Train: Iter=300 (0.036622 effective passes) train_loss=5.389 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.579 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=5.230 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=4.993 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=4.895 iter_dt=2.76e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=4.976 val_pp=144.854 val_acc=0.231894 +Train: Iter=550 (0.067140 effective passes) train_loss=4.982 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.615 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=4.969 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.818 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.819 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.633 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.536 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.493 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.566 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.589 iter_dt=2.74e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.477 val_pp=88.002 val_acc=0.276651 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.584 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.404 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.424 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.337 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.363 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.414 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.419 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.424 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.289 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.373 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.297 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.573 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.131 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.478 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.151 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.181 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.488 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.085 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.010 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.218 iter_dt=2.73e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.196 val_pp=66.403 val_acc=0.300983 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.210 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.234 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.182 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.171 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.219 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.931 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.119 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.121 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.129 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.360 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.140 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.940 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.351 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.230 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.133 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.989 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.274 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.192 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.141 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.002 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.159 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.783 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.039 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.096 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.169 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.036 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.252 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=3.955 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.926 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.026 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.098 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.155 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.109 iter_dt=2.90e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=3.990 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=3.971 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.089 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.914 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.021 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.063 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.108 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.012 iter_dt=2.74e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.015 val_pp=55.441 val_acc=0.316336 diff --git a/exps/sub_one_pass/formal/sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456/summary.json new file mode 100644 index 0000000..78eb9b6 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.356648921966553, 6.503230094909668, 6.109075546264648, 5.980000019073486, 5.80294132232666, 5.389494895935059, 5.579189300537109, 5.2298736572265625, 4.992541790008545, 4.8951416015625, 4.982321739196777, 4.614548683166504, 4.96866512298584, 4.818180084228516, 4.818872928619385, 4.633196830749512, 4.5363569259643555, 4.493485927581787, 4.5664215087890625, 4.589224815368652, 4.584003448486328, 4.404362678527832, 4.423949241638184, 4.336876392364502, 4.363476753234863, 4.4138569831848145, 4.418825149536133, 4.424135208129883, 4.2887959480285645, 4.372894287109375, 4.296518325805664, 4.573336601257324, 4.13141393661499, 4.477952480316162, 4.151226997375488, 4.181147575378418, 4.487954139709473, 4.085090637207031, 4.009576320648193, 4.218439102172852, 4.21036434173584, 4.233739376068115, 4.181670188903809, 4.170554161071777, 4.218930721282959, 3.9310855865478516, 4.119112014770508, 4.120658874511719, 4.129301071166992, 4.360270977020264, 4.139721870422363, 3.939685821533203, 4.351229667663574, 4.229901313781738, 4.132580757141113, 3.9886019229888916, 4.273507118225098, 4.191652297973633, 4.141087055206299, 4.001533508300781, 4.158641815185547, 3.783468246459961, 4.0391035079956055, 4.096146583557129, 4.169480800628662, 4.0363850593566895, 4.251852989196777, 3.9548399448394775, 3.9256818294525146, 4.025948524475098, 4.097853183746338, 4.154973983764648, 4.109405040740967, 3.9895262718200684, 3.9706127643585205, 4.088957786560059, 3.913571834564209, 4.020992755889893, 4.062801361083984, 4.10841178894043, 4.012358665466309], "val_loss": [5.664737701416016, 4.975729465484619, 4.477364540100098, 4.195748329162598, 4.015315055847168], "val_pp": [288.51119547894774, 144.85396812324217, 88.00217485099525, 66.40321718440944, 55.440609846288844], "val_acc": [0.18284988403320312, 0.23189353942871094, 0.27665138244628906, 0.3009834289550781, 0.31633567810058594], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.664737701416016, "val_perplexity": 288.51119547894774, "val_accuracy": 0.18284988403320312, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 4.975729465484619, "val_perplexity": 144.85396812324217, "val_accuracy": 0.23189353942871094, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.477364540100098, "val_perplexity": 88.00217485099525, "val_accuracy": 0.27665138244628906, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.195748329162598, "val_perplexity": 66.40321718440944, "val_accuracy": 0.3009834289550781, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.015315055847168, "val_perplexity": 55.440609846288844, "val_accuracy": 0.31633567810058594, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.276373394, 0.274558508, 0.275523371, 0.285118084, 0.273886871, 0.274745436, 0.27427533, 0.274946179, 0.275219348, 0.275528193, 0.273701766, 0.273348777, 0.273793523, 0.273356788, 0.273767738, 0.273827437, 0.27442328, 0.274501491, 0.274193391, 0.274088055, 0.273839821, 0.273913447, 0.274038716, 0.274719093, 0.274455725, 0.273071338, 0.273526947, 0.274411844, 0.273967344, 0.272748085, 0.274938334, 0.275407798, 0.27517778, 0.276824392, 0.275440671, 0.275611808, 0.276816915, 0.273050763, 0.273751673, 0.273461536, 0.274389756, 0.273811094, 0.274159107, 0.273590994, 0.273310922, 0.280094086, 0.274571495, 0.274637706, 0.273527237, 0.275580327, 0.274507878, 0.274210895, 0.273915843, 0.273809735, 0.272840102, 0.273458652, 0.273374609, 0.276588586, 0.274712239, 0.273535761, 0.273324185, 0.273017336, 0.273082266, 0.273505039, 0.274012852, 0.2730231, 0.273356058, 0.27599396, 0.273560366, 0.273644291, 0.275026209, 0.273115511, 0.289860934, 0.273373096, 0.273767627, 0.273058249, 0.275381103, 0.275285003, 0.273000772, 0.273093093, 0.27413812], "train_time_seconds": 1123.6289774160014, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 238900.43901975403, "peak_memory_bytes": 6314994176, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456", "seed": 2, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "ademamix", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.999, "adema_alpha": 8.0, "adema_beta3_warmup": 4096, "adema_alpha_warmup": 4096, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456/run.log new file mode 100644 index 0000000..f752f8c --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456/run.log @@ -0,0 +1,140 @@ +Starting Experiment: sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456', 'seed': 1, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 0.03, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Muon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + adamw_lr: 0.003 + adamw_lr_ratio: 0.1 + adamw_wd: 0.1 + lr: 0.03 + momentum: 0.95 + nesterov: True + ns_steps: 5 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.682 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=5.823 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=5.471 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=5.360 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=5.328 iter_dt=2.83e-01s lr=3.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.146 val_pp=171.719 val_acc=0.214575 +Train: Iter=300 (0.036622 effective passes) train_loss=4.927 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=350 (0.042725 effective passes) train_loss=5.135 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=400 (0.048829 effective passes) train_loss=4.751 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=450 (0.054932 effective passes) train_loss=4.569 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=500 (0.061036 effective passes) train_loss=4.622 iter_dt=2.83e-01s lr=3.00e-02 +>Eval: Iter=512 (0.062501 effective passes) val_loss=4.698 val_pp=109.707 val_acc=0.256454 +Train: Iter=550 (0.067140 effective passes) train_loss=4.687 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=600 (0.073243 effective passes) train_loss=4.450 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=650 (0.079347 effective passes) train_loss=4.789 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=700 (0.085451 effective passes) train_loss=4.683 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=750 (0.091554 effective passes) train_loss=4.691 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=800 (0.097658 effective passes) train_loss=4.537 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=850 (0.103761 effective passes) train_loss=4.450 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=900 (0.109865 effective passes) train_loss=4.402 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=950 (0.115969 effective passes) train_loss=4.485 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.523 iter_dt=2.84e-01s lr=3.00e-02 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.403 val_pp=81.679 val_acc=0.282814 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.534 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.342 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.365 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.306 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.303 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.358 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.374 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.382 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.262 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.324 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.278 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.552 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.121 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.461 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.132 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.177 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.463 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.068 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.010 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.203 iter_dt=2.88e-01s lr=3.00e-02 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.192 val_pp=66.174 val_acc=0.300663 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.209 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.216 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.177 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.187 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.207 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.941 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.128 iter_dt=2.82e-01s lr=3.00e-02 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.119 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.103 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.373 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.148 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.948 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.352 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.249 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.120 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.993 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.301 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.213 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.156 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.002 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.183 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.822 iter_dt=2.82e-01s lr=3.00e-02 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.057 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.109 iter_dt=2.82e-01s lr=3.00e-02 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.184 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.053 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.246 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3400 (0.415045 effective passes) train_loss=3.986 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.956 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.051 iter_dt=2.82e-01s lr=3.00e-02 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.112 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.197 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.135 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.017 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.000 iter_dt=2.82e-01s lr=3.00e-02 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.108 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.944 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.045 iter_dt=2.83e-01s lr=3.00e-02 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.106 iter_dt=2.82e-01s lr=3.00e-02 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.146 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.039 iter_dt=2.83e-01s lr=3.00e-02 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.044 val_pp=57.038 val_acc=0.313625 diff --git a/exps/sub_one_pass/formal/sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456/summary.json new file mode 100644 index 0000000..b0d80a5 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [6.681993007659912, 5.822784423828125, 5.471349239349365, 5.360372543334961, 5.328383922576904, 4.927132606506348, 5.135274887084961, 4.7511773109436035, 4.5685343742370605, 4.6217041015625, 4.687287330627441, 4.450047016143799, 4.788556098937988, 4.683255195617676, 4.690767288208008, 4.537054538726807, 4.450311660766602, 4.402149200439453, 4.484663963317871, 4.523004531860352, 4.5336174964904785, 4.341950416564941, 4.365231513977051, 4.306100845336914, 4.302720546722412, 4.357656002044678, 4.373667240142822, 4.381776809692383, 4.262297630310059, 4.324244499206543, 4.2779388427734375, 4.551996231079102, 4.121455669403076, 4.461276054382324, 4.132148265838623, 4.176900863647461, 4.463001251220703, 4.068317890167236, 4.010244846343994, 4.203333854675293, 4.208915710449219, 4.216151237487793, 4.177465438842773, 4.18719482421875, 4.2068705558776855, 3.940591335296631, 4.1279754638671875, 4.119321823120117, 4.103185653686523, 4.373233318328857, 4.147791862487793, 3.9475085735321045, 4.3520402908325195, 4.248552322387695, 4.119935512542725, 3.993014097213745, 4.301154136657715, 4.212721824645996, 4.156032562255859, 4.0017218589782715, 4.183412075042725, 3.822237968444824, 4.056743621826172, 4.1087846755981445, 4.1841959953308105, 4.05342960357666, 4.24576473236084, 3.9859719276428223, 3.9564528465270996, 4.05107307434082, 4.112186431884766, 4.197135925292969, 4.135276794433594, 4.0174736976623535, 3.9995334148406982, 4.108011245727539, 3.944286823272705, 4.045229434967041, 4.106058120727539, 4.146227836608887, 4.038932800292969], "val_loss": [5.1458635330200195, 4.69781494140625, 4.402804374694824, 4.192287445068359, 4.0437235832214355], "val_pp": [171.71911191532246, 109.70684704341419, 81.67936610685706, 66.17380072145147, 57.03817971687212], "val_acc": [0.21457481384277344, 0.2564544677734375, 0.28281402587890625, 0.3006629943847656, 0.3136253356933594], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.1458635330200195, "val_perplexity": 171.71911191532246, "val_accuracy": 0.21457481384277344, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 4.69781494140625, "val_perplexity": 109.70684704341419, "val_accuracy": 0.2564544677734375, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.402804374694824, "val_perplexity": 81.67936610685706, "val_accuracy": 0.28281402587890625, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.192287445068359, "val_perplexity": 66.17380072145147, "val_accuracy": 0.3006629943847656, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.0437235832214355, "val_perplexity": 57.03817971687212, "val_accuracy": 0.3136253356933594, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.282545273, 0.283473024, 0.286800223, 0.282826605, 0.283020383, 0.282988485, 0.282779556, 0.282552215, 0.28295112, 0.282672467, 0.283938719, 0.284142012, 0.282743035, 0.284720152, 0.286257155, 0.286178233, 0.285176663, 0.283648042, 0.283026418, 0.283707373, 0.283505963, 0.282952795, 0.283586945, 0.283755631, 0.283270847, 0.283907577, 0.284306489, 0.28405341, 0.282556548, 0.28367633, 0.283811358, 0.28341101, 0.283453728, 0.283734458, 0.28348575, 0.283310759, 0.282779228, 0.283891931, 0.286010399, 0.28797712, 0.283336053, 0.283658485, 0.283292372, 0.283433201, 0.283273362, 0.283490457, 0.28249791, 0.282504725, 0.28332553, 0.283551844, 0.286636865, 0.283338454, 0.284043434, 0.284475031, 0.282960229, 0.284080585, 0.283449948, 0.282898409, 0.282727637, 0.285896428, 0.283259563, 0.282217771, 0.283678377, 0.282399451, 0.283647319, 0.284464187, 0.287230509, 0.282821386, 0.283944646, 0.282462056, 0.283351498, 0.284023464, 0.284210149, 0.2832279, 0.282496974, 0.282978229, 0.28274838, 0.283117472, 0.282108985, 0.284157739, 0.283141246], "train_time_seconds": 1163.3150507089977, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 230750.43672511456, "peak_memory_bytes": 6117368832, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456", "seed": 1, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "muon", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.95, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": true, "muon_ns_steps": 5, "muon_lr_factor": 0.03, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456/run.log new file mode 100644 index 0000000..0559ff4 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456/run.log @@ -0,0 +1,140 @@ +Starting Experiment: sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456', 'seed': 2, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 0.03, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Muon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + adamw_lr: 0.003 + adamw_lr_ratio: 0.1 + adamw_wd: 0.1 + lr: 0.03 + momentum: 0.95 + nesterov: True + ns_steps: 5 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.661 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=5.798 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=5.455 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=5.351 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=5.319 iter_dt=2.90e-01s lr=3.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.148 val_pp=172.119 val_acc=0.214087 +Train: Iter=300 (0.036622 effective passes) train_loss=4.954 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=350 (0.042725 effective passes) train_loss=5.123 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=400 (0.048829 effective passes) train_loss=4.758 iter_dt=2.91e-01s lr=3.00e-02 +Train: Iter=450 (0.054932 effective passes) train_loss=4.578 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=500 (0.061036 effective passes) train_loss=4.608 iter_dt=2.86e-01s lr=3.00e-02 +>Eval: Iter=512 (0.062501 effective passes) val_loss=4.698 val_pp=109.759 val_acc=0.256956 +Train: Iter=550 (0.067140 effective passes) train_loss=4.688 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=600 (0.073243 effective passes) train_loss=4.434 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=650 (0.079347 effective passes) train_loss=4.785 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=700 (0.085451 effective passes) train_loss=4.679 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=750 (0.091554 effective passes) train_loss=4.682 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=800 (0.097658 effective passes) train_loss=4.516 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=850 (0.103761 effective passes) train_loss=4.439 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=900 (0.109865 effective passes) train_loss=4.396 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=950 (0.115969 effective passes) train_loss=4.472 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.503 iter_dt=2.89e-01s lr=3.00e-02 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.398 val_pp=81.320 val_acc=0.282288 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.515 iter_dt=2.91e-01s lr=3.00e-02 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.335 iter_dt=2.91e-01s lr=3.00e-02 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.370 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.285 iter_dt=2.91e-01s lr=3.00e-02 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.318 iter_dt=2.99e-01s lr=3.00e-02 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.357 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.364 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.386 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.257 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.329 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.276 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.545 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.105 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.445 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.148 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.160 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.463 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.060 iter_dt=2.72e-01s lr=3.00e-02 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.015 iter_dt=2.91e-01s lr=3.00e-02 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.206 iter_dt=2.94e-01s lr=3.00e-02 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.190 val_pp=66.034 val_acc=0.300352 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.203 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.215 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.193 iter_dt=2.92e-01s lr=3.00e-02 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.187 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.210 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.916 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.129 iter_dt=2.92e-01s lr=3.00e-02 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.123 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.106 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.374 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.155 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.957 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.366 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.259 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.132 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.995 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.299 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.208 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.156 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=3000 (0.366217 effective passes) train_loss=3.998 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.189 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.806 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.029 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.116 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.181 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.044 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.259 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3400 (0.415045 effective passes) train_loss=3.990 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.952 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.056 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.126 iter_dt=2.91e-01s lr=3.00e-02 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.185 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.148 iter_dt=2.91e-01s lr=3.00e-02 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.032 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.007 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.120 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.956 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.045 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.117 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.125 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.046 iter_dt=2.86e-01s lr=3.00e-02 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.044 val_pp=57.070 val_acc=0.313566 diff --git a/exps/sub_one_pass/formal/sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456/summary.json new file mode 100644 index 0000000..1493212 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [6.661473751068115, 5.797703266143799, 5.455463409423828, 5.351103782653809, 5.318686485290527, 4.9544878005981445, 5.123226165771484, 4.757715225219727, 4.577827453613281, 4.607831954956055, 4.68833065032959, 4.433945655822754, 4.784580230712891, 4.679389953613281, 4.682063102722168, 4.515847206115723, 4.438681602478027, 4.395623683929443, 4.472327709197998, 4.502992630004883, 4.514847755432129, 4.334821701049805, 4.369668960571289, 4.2848615646362305, 4.317702293395996, 4.356830596923828, 4.363691329956055, 4.385512351989746, 4.256684303283691, 4.329447269439697, 4.276451587677002, 4.544639587402344, 4.104504108428955, 4.44540548324585, 4.147903919219971, 4.160171985626221, 4.462949752807617, 4.060003280639648, 4.015449523925781, 4.206235885620117, 4.203107833862305, 4.21486759185791, 4.193182468414307, 4.186517238616943, 4.209789276123047, 3.916104555130005, 4.128525733947754, 4.1228532791137695, 4.105685234069824, 4.374334335327148, 4.154945373535156, 3.957496166229248, 4.365741729736328, 4.258790016174316, 4.132420063018799, 3.9947595596313477, 4.299006462097168, 4.208180904388428, 4.156343936920166, 3.9982213973999023, 4.189235687255859, 3.805544137954712, 4.029394149780273, 4.116138935089111, 4.181478023529053, 4.043934345245361, 4.2594523429870605, 3.9898552894592285, 3.951805591583252, 4.056143283843994, 4.126155376434326, 4.1850714683532715, 4.14822244644165, 4.031571388244629, 4.006633281707764, 4.120169639587402, 3.955554723739624, 4.045009613037109, 4.116947174072266, 4.125340461730957, 4.0456671714782715], "val_loss": [5.148187637329102, 4.698286056518555, 4.398398399353027, 4.190174102783203, 4.04427433013916], "val_pp": [172.1186689007199, 109.75854373879675, 81.32028071698167, 66.03410059327533, 57.06960194946216], "val_acc": [0.21408653259277344, 0.2569561004638672, 0.28228759765625, 0.3003520965576172, 0.3135662078857422], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.148187637329102, "val_perplexity": 172.1186689007199, "val_accuracy": 0.21408653259277344, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 4.698286056518555, "val_perplexity": 109.75854373879675, "val_accuracy": 0.2569561004638672, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.398398399353027, "val_perplexity": 81.32028071698167, "val_accuracy": 0.28228759765625, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.190174102783203, "val_perplexity": 66.03410059327533, "val_accuracy": 0.3003520965576172, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.04427433013916, "val_perplexity": 57.06960194946216, "val_accuracy": 0.3135662078857422, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.286457551, 0.287367515, 0.287678873, 0.287996662, 0.289503775, 0.287757845, 0.287326655, 0.291148979, 0.289803758, 0.285617503, 0.285113236, 0.287586676, 0.289197273, 0.289296914, 0.287132569, 0.288927946, 0.287931382, 0.288249057, 0.288931526, 0.289071248, 0.291185303, 0.290600338, 0.290028265, 0.290820154, 0.298853674, 0.289086473, 0.289656946, 0.289295208, 0.286066741, 0.289897175, 0.287973658, 0.289388913, 0.288755469, 0.289108822, 0.287619571, 0.287519757, 0.29001795, 0.272180365, 0.290815446, 0.293724443, 0.289652506, 0.287215693, 0.292489587, 0.288338923, 0.28828609, 0.289392011, 0.291632809, 0.289232315, 0.289400058, 0.285215485, 0.285651892, 0.284408927, 0.284534448, 0.28899627, 0.286562352, 0.286980628, 0.28583598, 0.286277488, 0.285865536, 0.2845128, 0.287448755, 0.284810455, 0.287273012, 0.289913886, 0.287457938, 0.285090121, 0.286915013, 0.286083791, 0.286149646, 0.287236865, 0.290830323, 0.286501946, 0.291446892, 0.289947807, 0.289033345, 0.287869517, 0.287079965, 0.285885716, 0.287263591, 0.286155544, 0.285958725], "train_time_seconds": 1180.8367941000026, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 227326.46657118542, "peak_memory_bytes": 6117368832, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456", "seed": 2, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "muon", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.95, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": true, "muon_ns_steps": 5, "muon_lr_factor": 0.03, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456/run.log new file mode 100644 index 0000000..8d7bdbc --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456/run.log @@ -0,0 +1,155 @@ +Starting Experiment: sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456', 'seed': 1, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'soap', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SOAP ( +Parameter Group 0 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.001 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.001 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.375 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.148 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=5.711 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.573 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.462 iter_dt=2.80e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.308 val_pp=202.004 val_acc=0.208868 +Train: Iter=300 (0.036622 effective passes) train_loss=5.136 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.281 iter_dt=2.87e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=4.894 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=4.646 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=4.634 iter_dt=2.80e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=4.715 val_pp=111.628 val_acc=0.256311 +Train: Iter=550 (0.067140 effective passes) train_loss=4.694 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.424 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=4.770 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.661 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.659 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.501 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.433 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.379 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.448 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.491 iter_dt=2.79e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.378 val_pp=79.661 val_acc=0.283842 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.501 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.300 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.345 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.271 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.297 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.342 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.345 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.383 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.242 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.309 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.249 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.533 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.111 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.458 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.125 iter_dt=2.88e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.142 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.444 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.062 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=3.991 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.214 iter_dt=2.80e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.176 val_pp=65.095 val_acc=0.299778 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.185 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.203 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.168 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.158 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.207 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.902 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.102 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.106 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.101 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.366 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.135 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.941 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.362 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.237 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.117 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.981 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.282 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.209 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.152 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=3.993 iter_dt=2.90e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.178 iter_dt=2.88e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.798 iter_dt=3.00e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.028 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.105 iter_dt=2.97e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.166 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.053 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.246 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=3.985 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.939 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.047 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.106 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.185 iter_dt=2.89e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.137 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.017 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=3.991 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.095 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.946 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.033 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.098 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.135 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.031 iter_dt=2.81e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.037 val_pp=56.676 val_acc=0.312168 diff --git a/exps/sub_one_pass/formal/sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456/summary.json new file mode 100644 index 0000000..f308083 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.375171184539795, 6.14778995513916, 5.7111053466796875, 5.572808742523193, 5.461978912353516, 5.135895729064941, 5.281177520751953, 4.893805503845215, 4.646465301513672, 4.6336469650268555, 4.6937336921691895, 4.42378568649292, 4.770384788513184, 4.660739421844482, 4.6588029861450195, 4.500690460205078, 4.432692527770996, 4.379134178161621, 4.447942733764648, 4.490841865539551, 4.500558853149414, 4.299563407897949, 4.345428943634033, 4.27146053314209, 4.296911239624023, 4.341573238372803, 4.345360279083252, 4.383098602294922, 4.241607666015625, 4.308701515197754, 4.248786926269531, 4.533186912536621, 4.111340045928955, 4.457643508911133, 4.125331401824951, 4.141813278198242, 4.444156169891357, 4.062260150909424, 3.990950107574463, 4.213517189025879, 4.184797763824463, 4.203361988067627, 4.167778968811035, 4.1582417488098145, 4.206908226013184, 3.902033805847168, 4.102285385131836, 4.105727195739746, 4.100741386413574, 4.366457939147949, 4.134737968444824, 3.9411563873291016, 4.362332344055176, 4.237165451049805, 4.116578102111816, 3.9811043739318848, 4.281851768493652, 4.209436893463135, 4.152273654937744, 3.9927287101745605, 4.177646160125732, 3.7979397773742676, 4.028470039367676, 4.105344772338867, 4.165692329406738, 4.0525617599487305, 4.245977401733398, 3.985189437866211, 3.939319133758545, 4.046818733215332, 4.105874061584473, 4.184961318969727, 4.1369781494140625, 4.016590118408203, 3.9909024238586426, 4.0945725440979, 3.946058750152588, 4.032625198364258, 4.09830904006958, 4.134759902954102, 4.031430244445801], "val_loss": [5.308291912078857, 4.7151780128479, 4.377788543701172, 4.175849437713623, 4.037361145019531], "val_pp": [202.0041701392595, 111.62832669491085, 79.66143558302677, 65.09492757975597, 56.67643009174277], "val_acc": [0.20886802673339844, 0.25631141662597656, 0.2838420867919922, 0.2997779846191406, 0.3121681213378906], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.308291912078857, "val_perplexity": 202.0041701392595, "val_accuracy": 0.20886802673339844, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 4.7151780128479, "val_perplexity": 111.62832669491085, "val_accuracy": 0.25631141662597656, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.377788543701172, "val_perplexity": 79.66143558302677, "val_accuracy": 0.2838420867919922, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.175849437713623, "val_perplexity": 65.09492757975597, "val_accuracy": 0.2997779846191406, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.037361145019531, "val_perplexity": 56.67643009174277, "val_accuracy": 0.3121681213378906, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.285048725, 0.28422132, 0.280834924, 0.278863945, 0.279536457, 0.27877385, 0.287380566, 0.279162599, 0.278910887, 0.279907765, 0.280795129, 0.279098452, 0.278683182, 0.279494658, 0.279622656, 0.279880275, 0.279329508, 0.278628574, 0.278511812, 0.279412802, 0.27644739, 0.280717547, 0.28047287, 0.278680195, 0.278681946, 0.278302595, 0.279792913, 0.27918184, 0.282588351, 0.278966378, 0.280305836, 0.280058015, 0.280358989, 0.285408121, 0.288104744, 0.284457435, 0.284448362, 0.281128544, 0.279365869, 0.279751378, 0.278907625, 0.280717885, 0.280267602, 0.28048921, 0.278950669, 0.280112989, 0.280578202, 0.281114115, 0.27959812, 0.278591791, 0.279196323, 0.278981417, 0.280922541, 0.279697776, 0.280548013, 0.279591997, 0.279642595, 0.278547923, 0.279667641, 0.28962275, 0.288395509, 0.300131426, 0.280296008, 0.296995715, 0.279431433, 0.280977955, 0.281848163, 0.278838121, 0.278973273, 0.278712339, 0.280409795, 0.289155025, 0.280431892, 0.278409051, 0.279683515, 0.278905752, 0.284093354, 0.278788385, 0.278219092, 0.279581872, 0.280913536], "train_time_seconds": 1276.103207307998, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 210355.6001291445, "peak_memory_bytes": 6559245312, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456", "seed": 1, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "soap", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456/run.log new file mode 100644 index 0000000..88137d8 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456/run.log @@ -0,0 +1,155 @@ +Starting Experiment: sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456', 'seed': 2, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'soap', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SOAP ( +Parameter Group 0 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.001 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.001 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.438 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.148 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=5.693 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.573 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.449 iter_dt=2.84e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.292 val_pp=198.685 val_acc=0.210087 +Train: Iter=300 (0.036622 effective passes) train_loss=5.132 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.280 iter_dt=2.87e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=4.871 iter_dt=2.89e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=4.619 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=4.631 iter_dt=2.84e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=4.710 val_pp=111.081 val_acc=0.257057 +Train: Iter=550 (0.067140 effective passes) train_loss=4.671 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.424 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=4.764 iter_dt=2.86e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.663 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.676 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.492 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.422 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.383 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.446 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.483 iter_dt=2.83e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.380 val_pp=79.876 val_acc=0.283529 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.506 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.300 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.343 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.276 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.299 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.342 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.366 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.383 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.228 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.319 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.257 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.537 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.106 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.449 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.141 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.170 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.461 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.073 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=3.996 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.208 iter_dt=2.80e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.179 val_pp=65.316 val_acc=0.300276 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.195 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.195 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.156 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.165 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.212 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.919 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.119 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.106 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.115 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.360 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.150 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.943 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.357 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.229 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.125 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.976 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.283 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.190 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.146 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.000 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.187 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.805 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.038 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.102 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.170 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.056 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.270 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=3.978 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.955 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.033 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.109 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.200 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.125 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.016 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=3.987 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.098 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.939 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.035 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.102 iter_dt=2.95e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.143 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.031 iter_dt=2.80e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.037 val_pp=56.663 val_acc=0.312256 diff --git a/exps/sub_one_pass/formal/sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456/summary.json new file mode 100644 index 0000000..bbe47cb --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.438036918640137, 6.14779806137085, 5.693320274353027, 5.572917938232422, 5.448721408843994, 5.131532669067383, 5.279992580413818, 4.871400833129883, 4.6185150146484375, 4.63094425201416, 4.6713714599609375, 4.423504829406738, 4.764473915100098, 4.662771224975586, 4.67603874206543, 4.4920454025268555, 4.421943187713623, 4.382859230041504, 4.446420669555664, 4.48307991027832, 4.505770683288574, 4.2998366355896, 4.342954635620117, 4.276186466217041, 4.29888916015625, 4.341660499572754, 4.366393566131592, 4.383081436157227, 4.2275896072387695, 4.318785667419434, 4.257282733917236, 4.537351608276367, 4.106178283691406, 4.448643207550049, 4.1409502029418945, 4.16963005065918, 4.460939407348633, 4.073092460632324, 3.9959330558776855, 4.208288192749023, 4.19462776184082, 4.195018768310547, 4.156408786773682, 4.16453742980957, 4.212496280670166, 3.9190430641174316, 4.118943214416504, 4.106478691101074, 4.114590644836426, 4.360465049743652, 4.150341033935547, 3.9434213638305664, 4.356971263885498, 4.229069709777832, 4.125079154968262, 3.9761345386505127, 4.282809257507324, 4.190092086791992, 4.146365165710449, 4.000399112701416, 4.187226295471191, 3.8045530319213867, 4.0377044677734375, 4.102055549621582, 4.169573783874512, 4.056116104125977, 4.270054817199707, 3.9775991439819336, 3.955470561981201, 4.032627105712891, 4.109036922454834, 4.199503421783447, 4.125377655029297, 4.016075134277344, 3.9868435859680176, 4.098315715789795, 3.9388294219970703, 4.035094261169434, 4.102286338806152, 4.1429853439331055, 4.031181812286377], "val_loss": [5.2917256355285645, 4.710265636444092, 4.380484580993652, 4.179245948791504, 4.037126541137695], "val_pp": [198.68528213612078, 111.08131137740004, 79.87649541386698, 65.31639897553266, 56.66313514976101], "val_acc": [0.21008682250976562, 0.25705718994140625, 0.28352928161621094, 0.3002758026123047, 0.312255859375], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.2917256355285645, "val_perplexity": 198.68528213612078, "val_accuracy": 0.21008682250976562, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 4.710265636444092, "val_perplexity": 111.08131137740004, "val_accuracy": 0.25705718994140625, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.380484580993652, "val_perplexity": 79.87649541386698, "val_accuracy": 0.28352928161621094, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.179245948791504, "val_perplexity": 65.31639897553266, "val_accuracy": 0.3002758026123047, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.037126541137695, "val_perplexity": 56.66313514976101, "val_accuracy": 0.312255859375, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.282313376, 0.284156886, 0.284929627, 0.282589749, 0.284200539, 0.283479104, 0.287497611, 0.288625131, 0.284590457, 0.284252969, 0.28483716, 0.285176348, 0.285973251, 0.283349319, 0.285240598, 0.283501717, 0.28493635, 0.284255651, 0.285257781, 0.283028002, 0.282687338, 0.28106655, 0.280683536, 0.280730116, 0.281437204, 0.28084102, 0.280821885, 0.281327188, 0.280646931, 0.280712055, 0.278435947, 0.279602426, 0.283134561, 0.280318049, 0.280909109, 0.280766381, 0.280615715, 0.279796338, 0.279823082, 0.279833788, 0.280058169, 0.280474698, 0.280393671, 0.279695434, 0.280102812, 0.280201197, 0.279897392, 0.279749665, 0.278775919, 0.280827933, 0.280758523, 0.280627318, 0.281033323, 0.280225736, 0.279408616, 0.279928014, 0.280022466, 0.279657089, 0.279384818, 0.280281793, 0.279799478, 0.280234858, 0.28075043, 0.280749122, 0.281202475, 0.281763242, 0.280898727, 0.28028274, 0.281940367, 0.280456836, 0.282150598, 0.281864601, 0.278239345, 0.280040787, 0.281514356, 0.279628262, 0.280638511, 0.28265484, 0.295335622, 0.281122897, 0.279963347], "train_time_seconds": 1281.7409867240024, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 209430.34418060805, "peak_memory_bytes": 6559245312, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456", "seed": 2, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "soap", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456/run.log new file mode 100644 index 0000000..1be50e6 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456/run.log @@ -0,0 +1,151 @@ +Starting Experiment: sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adafactor', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Adafactor ( +Parameter Group 0 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.0003 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.1 + +Parameter Group 1 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.0003 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.464 iter_dt=2.90e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.453 iter_dt=2.92e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.160 iter_dt=3.00e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.137 iter_dt=2.93e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.004 iter_dt=2.92e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.914 val_pp=370.206 val_acc=0.181087 +Train: Iter=300 (0.036622 effective passes) train_loss=5.816 iter_dt=2.96e-01s lr=3.00e-04 +Train: Iter=350 (0.042725 effective passes) train_loss=5.921 iter_dt=2.95e-01s lr=3.00e-04 +Train: Iter=400 (0.048829 effective passes) train_loss=5.620 iter_dt=2.95e-01s lr=3.00e-04 +Train: Iter=450 (0.054932 effective passes) train_loss=5.434 iter_dt=2.94e-01s lr=3.00e-04 +Train: Iter=500 (0.061036 effective passes) train_loss=5.331 iter_dt=2.94e-01s lr=3.00e-04 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.436 val_pp=229.608 val_acc=0.204878 +Train: Iter=550 (0.067140 effective passes) train_loss=5.562 iter_dt=2.91e-01s lr=3.00e-04 +Train: Iter=600 (0.073243 effective passes) train_loss=5.035 iter_dt=2.94e-01s lr=3.00e-04 +Train: Iter=650 (0.079347 effective passes) train_loss=5.509 iter_dt=2.93e-01s lr=3.00e-04 +Train: Iter=700 (0.085451 effective passes) train_loss=5.316 iter_dt=2.90e-01s lr=3.00e-04 +Train: Iter=750 (0.091554 effective passes) train_loss=5.292 iter_dt=2.91e-01s lr=3.00e-04 +Train: Iter=800 (0.097658 effective passes) train_loss=5.076 iter_dt=2.91e-01s lr=3.00e-04 +Train: Iter=850 (0.103761 effective passes) train_loss=5.093 iter_dt=2.93e-01s lr=3.00e-04 +Train: Iter=900 (0.109865 effective passes) train_loss=5.043 iter_dt=2.96e-01s lr=3.00e-04 +Train: Iter=950 (0.115969 effective passes) train_loss=5.051 iter_dt=2.95e-01s lr=3.00e-04 +Train: Iter=1000 (0.122072 effective passes) train_loss=5.027 iter_dt=3.04e-01s lr=3.00e-04 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=5.031 val_pp=153.055 val_acc=0.227949 +Train: Iter=1050 (0.128176 effective passes) train_loss=5.124 iter_dt=2.93e-01s lr=3.00e-04 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.976 iter_dt=2.95e-01s lr=3.00e-04 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.898 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.868 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.842 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.979 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.893 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.870 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.769 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.865 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.711 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1600 (0.195315 effective passes) train_loss=5.032 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.572 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.922 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.612 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.624 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.904 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.510 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.489 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.659 iter_dt=2.85e-01s lr=3.00e-04 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.612 val_pp=100.683 val_acc=0.264980 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.640 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.629 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.575 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.537 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.598 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.342 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.485 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.532 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.490 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.745 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.547 iter_dt=2.91e-01s lr=3.00e-04 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.303 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.680 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.601 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.474 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.326 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.651 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.570 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.471 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.330 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.506 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3100 (0.378424 effective passes) train_loss=4.097 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.369 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.427 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.496 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.385 iter_dt=2.89e-01s lr=3.00e-04 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.576 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.283 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.251 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.330 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.415 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.494 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.438 iter_dt=2.91e-01s lr=3.00e-04 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.291 iter_dt=2.89e-01s lr=3.00e-04 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.306 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.414 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.210 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.315 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.392 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.447 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.298 iter_dt=2.86e-01s lr=3.00e-04 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.321 val_pp=75.290 val_acc=0.289244 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456/summary.json new file mode 100644 index 0000000..4230774 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.464272499084473, 6.453136920928955, 6.1597185134887695, 6.137300491333008, 6.003512859344482, 5.816338539123535, 5.920787811279297, 5.6199235916137695, 5.433855056762695, 5.331022262573242, 5.562030792236328, 5.03509521484375, 5.509119987487793, 5.315619945526123, 5.29201602935791, 5.075938701629639, 5.093226432800293, 5.042997360229492, 5.050856590270996, 5.027212142944336, 5.124483108520508, 4.976062774658203, 4.898069381713867, 4.868346214294434, 4.841808319091797, 4.978519439697266, 4.893424987792969, 4.8699822425842285, 4.769359588623047, 4.864572525024414, 4.710756301879883, 5.0315446853637695, 4.572122097015381, 4.922338485717773, 4.611930847167969, 4.624235153198242, 4.903501510620117, 4.510046482086182, 4.489289283752441, 4.658594608306885, 4.640075206756592, 4.628575801849365, 4.574565887451172, 4.536843299865723, 4.598055839538574, 4.341836929321289, 4.48529052734375, 4.532341480255127, 4.490438461303711, 4.745311737060547, 4.546902179718018, 4.303002834320068, 4.680400371551514, 4.601487159729004, 4.474143028259277, 4.326450347900391, 4.651203155517578, 4.569860458374023, 4.471133232116699, 4.330431938171387, 4.50621223449707, 4.097294807434082, 4.368588447570801, 4.427144527435303, 4.496129035949707, 4.3849639892578125, 4.575713634490967, 4.2829413414001465, 4.251306533813477, 4.330356121063232, 4.414665222167969, 4.493817329406738, 4.438488006591797, 4.290544509887695, 4.306050777435303, 4.413670539855957, 4.210136413574219, 4.315332412719727, 4.39217472076416, 4.447185516357422, 4.298017978668213], "val_loss": [5.914063930511475, 5.436375141143799, 5.0308027267456055, 4.611979007720947, 4.321353912353516], "val_pp": [370.2061276976988, 229.60753557404612, 153.05530746366406, 100.6828931008601, 75.29027719728664], "val_acc": [0.18108749389648438, 0.2048778533935547, 0.2279491424560547, 0.2649803161621094, 0.2892436981201172], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.914063930511475, "val_perplexity": 370.2061276976988, "val_accuracy": 0.18108749389648438, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 5.436375141143799, "val_perplexity": 229.60753557404612, "val_accuracy": 0.2048778533935547, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 5.0308027267456055, "val_perplexity": 153.05530746366406, "val_accuracy": 0.2279491424560547, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.611979007720947, "val_perplexity": 100.6828931008601, "val_accuracy": 0.2649803161621094, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.321353912353516, "val_perplexity": 75.29027719728664, "val_accuracy": 0.2892436981201172, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.290131174, 0.292357494, 0.299839819, 0.293148398, 0.292160615, 0.296082029, 0.294567789, 0.294921024, 0.294425142, 0.293667266, 0.291146797, 0.294146947, 0.293096012, 0.289586053, 0.290562128, 0.290682091, 0.292541618, 0.296157395, 0.294585943, 0.303510341, 0.293200602, 0.294851056, 0.286445619, 0.286426846, 0.286193216, 0.28660248, 0.286278909, 0.286399499, 0.28619354, 0.285873787, 0.286241413, 0.286992545, 0.288389129, 0.286999057, 0.286078635, 0.284872631, 0.284915899, 0.284827625, 0.286350369, 0.285164177, 0.285276475, 0.284958015, 0.287347363, 0.285923598, 0.286355273, 0.286151624, 0.286796048, 0.285286846, 0.285638911, 0.285509255, 0.291407392, 0.287152661, 0.285581625, 0.285383625, 0.288316797, 0.287185753, 0.287912692, 0.286427823, 0.28663593, 0.284776185, 0.284666903, 0.285518582, 0.285062993, 0.285642307, 0.286390192, 0.288576651, 0.284669196, 0.286067426, 0.285631335, 0.288151474, 0.286488932, 0.286821597, 0.29074053, 0.289149659, 0.284883036, 0.28618385, 0.285833558, 0.288472792, 0.287383416, 0.286281089, 0.285818276], "train_time_seconds": 1182.1543548539998, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 227073.10166205216, "peak_memory_bytes": 6040051712, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "adafactor", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456/run.log new file mode 100644 index 0000000..41e273a --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456/run.log @@ -0,0 +1,156 @@ +Starting Experiment: sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +using fused AdamW: True + +Optimizer: +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.001 + maximize: False + weight_decay: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.001 + maximize: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.310 iter_dt=2.66e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.401 iter_dt=2.66e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.041 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.931 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.765 iter_dt=2.69e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.626 val_pp=277.625 val_acc=0.185545 +Train: Iter=300 (0.036622 effective passes) train_loss=5.397 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.556 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=5.272 iter_dt=2.66e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=5.019 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=4.976 iter_dt=2.67e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.043 val_pp=154.935 val_acc=0.224113 +Train: Iter=550 (0.067140 effective passes) train_loss=5.079 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.652 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=5.013 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.886 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.867 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.665 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.612 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.558 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.602 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.635 iter_dt=2.69e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.545 val_pp=94.133 val_acc=0.271013 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.642 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.455 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.481 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.411 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.429 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.467 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.493 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.484 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.345 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.425 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.356 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.624 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.200 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.546 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.203 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.254 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.545 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.148 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.107 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.295 iter_dt=2.68e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.271 val_pp=71.603 val_acc=0.293463 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.286 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.291 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.259 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.247 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.298 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.990 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.190 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.180 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.175 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.438 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.216 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.993 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.446 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.310 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.214 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.047 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.354 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.265 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.216 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.058 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.225 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.859 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.093 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.177 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.237 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.109 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.313 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.032 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.005 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.093 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.165 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.255 iter_dt=2.64e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.177 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.061 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.064 iter_dt=2.68e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.158 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.985 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.094 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.142 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.200 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.077 iter_dt=2.69e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.083 val_pp=59.349 val_acc=0.308804 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456/summary.json new file mode 100644 index 0000000..d0c6ab2 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.310066223144531, 6.4013261795043945, 6.041391372680664, 5.930922508239746, 5.765065670013428, 5.396886825561523, 5.555549621582031, 5.271737575531006, 5.019238471984863, 4.97576904296875, 5.078512668609619, 4.6518235206604, 5.013322830200195, 4.886430740356445, 4.86729621887207, 4.664588928222656, 4.61164665222168, 4.557901859283447, 4.601781845092773, 4.635075569152832, 4.641597747802734, 4.455051422119141, 4.481429100036621, 4.410677909851074, 4.428679943084717, 4.467085838317871, 4.493122577667236, 4.483615875244141, 4.3447113037109375, 4.4249725341796875, 4.356327056884766, 4.624362945556641, 4.200328826904297, 4.545588493347168, 4.202899932861328, 4.254039764404297, 4.545076370239258, 4.147973537445068, 4.107240200042725, 4.295398712158203, 4.285777568817139, 4.291470527648926, 4.258543014526367, 4.247244834899902, 4.297714710235596, 3.989769458770752, 4.1897993087768555, 4.179544448852539, 4.174990653991699, 4.437537670135498, 4.216202735900879, 3.9925999641418457, 4.445757865905762, 4.309511184692383, 4.213640213012695, 4.046568870544434, 4.353776454925537, 4.265326499938965, 4.215559959411621, 4.058409690856934, 4.225433826446533, 3.85909366607666, 4.092889785766602, 4.177485466003418, 4.236898422241211, 4.109156608581543, 4.31343936920166, 4.032168388366699, 4.004990577697754, 4.093198776245117, 4.165159702301025, 4.254501819610596, 4.1770339012146, 4.060903549194336, 4.064353942871094, 4.1577043533325195, 3.9846878051757812, 4.094124794006348, 4.142324447631836, 4.200030326843262, 4.07651424407959], "val_loss": [5.626274585723877, 5.043008327484131, 4.544707298278809, 4.271139621734619, 4.083443641662598], "val_pp": [277.6248664569638, 154.93488552851235, 94.13258255880584, 71.60298397602084, 59.34933373277334], "val_acc": [0.1855449676513672, 0.22411346435546875, 0.2710132598876953, 0.29346275329589844, 0.3088035583496094], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.626274585723877, "val_perplexity": 277.6248664569638, "val_accuracy": 0.1855449676513672, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 5.043008327484131, "val_perplexity": 154.93488552851235, "val_accuracy": 0.22411346435546875, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.544707298278809, "val_perplexity": 94.13258255880584, "val_accuracy": 0.2710132598876953, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.271139621734619, "val_perplexity": 71.60298397602084, "val_accuracy": 0.29346275329589844, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.083443641662598, "val_perplexity": 59.34933373277334, "val_accuracy": 0.3088035583496094, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.266388889, 0.265600652, 0.271355176, 0.268673289, 0.269405406, 0.268582031, 0.269428123, 0.265771578, 0.267094289, 0.267055556, 0.268303908, 0.26749313, 0.267848421, 0.267008748, 0.267367597, 0.268512586, 0.267668211, 0.267326203, 0.267114946, 0.268780874, 0.270038569, 0.267573377, 0.267220816, 0.267394983, 0.266712481, 0.266769469, 0.267614054, 0.267863198, 0.267601763, 0.26764542, 0.270802575, 0.267267914, 0.268526428, 0.268429215, 0.268374703, 0.268267878, 0.267747864, 0.268545325, 0.268133305, 0.267631221, 0.268863612, 0.270509217, 0.268935927, 0.269003704, 0.268465266, 0.269416795, 0.270512675, 0.268655486, 0.266565692, 0.268904082, 0.267872345, 0.267744346, 0.267438478, 0.267998581, 0.2688092, 0.267777004, 0.268244608, 0.268358663, 0.273378061, 0.272366268, 0.270517646, 0.271350785, 0.268750808, 0.268336737, 0.268381099, 0.267928762, 0.267925374, 0.26950108, 0.267619595, 0.267823368, 0.267135673, 0.263788479, 0.268753591, 0.269364051, 0.267663054, 0.269076814, 0.268531946, 0.268633369, 0.270664859, 0.268662146, 0.2685458], "train_time_seconds": 1098.9983692469998, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 244254.64451227876, "peak_memory_bytes": 6175430656, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "adamw", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456/run.log new file mode 100644 index 0000000..4fd6ebe --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456/run.log @@ -0,0 +1,147 @@ +Starting Experiment: sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'ademamix', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.999, 'adema_alpha': 8.0, 'adema_beta3_warmup': 4096, 'adema_alpha_warmup': 4096, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdEMAMix ( +Parameter Group 0 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.001 + weight_decay: 0.1 + +Parameter Group 1 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.001 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.303 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.491 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.144 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.979 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.822 iter_dt=2.76e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.677 val_pp=292.054 val_acc=0.181980 +Train: Iter=300 (0.036622 effective passes) train_loss=5.413 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.595 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=5.294 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=5.083 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=4.992 iter_dt=2.78e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.065 val_pp=158.348 val_acc=0.220623 +Train: Iter=550 (0.067140 effective passes) train_loss=5.108 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.668 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=5.023 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.867 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.852 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.660 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.561 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.506 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.572 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.591 iter_dt=2.75e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.498 val_pp=89.869 val_acc=0.274141 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.617 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.431 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.441 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.365 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.383 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.432 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.425 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.452 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.298 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.382 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.292 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.572 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.142 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.495 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.165 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.181 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.489 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.077 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.032 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.236 iter_dt=2.74e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.206 val_pp=67.078 val_acc=0.299908 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.220 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.234 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.190 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.176 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.235 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.932 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.118 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.110 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.117 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.385 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.161 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.943 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.368 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.238 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.117 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.996 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.293 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.201 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.152 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=3.995 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.171 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.800 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.038 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.115 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.158 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.026 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.240 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=3.966 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.951 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.018 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.101 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.172 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.111 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.009 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=3.984 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.080 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.916 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.021 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.096 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.108 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.019 iter_dt=2.74e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.020 val_pp=55.698 val_acc=0.316416 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456/summary.json new file mode 100644 index 0000000..09cd931 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.303215980529785, 6.490687847137451, 6.1435346603393555, 5.978877544403076, 5.821807861328125, 5.4129109382629395, 5.59468412399292, 5.294363498687744, 5.082927703857422, 4.992424011230469, 5.10750150680542, 4.668220520019531, 5.0229387283325195, 4.8666791915893555, 4.851797103881836, 4.6600799560546875, 4.560825347900391, 4.506239414215088, 4.572469711303711, 4.591252326965332, 4.616550922393799, 4.431182861328125, 4.441056728363037, 4.365049362182617, 4.383415222167969, 4.432480812072754, 4.424840927124023, 4.452183246612549, 4.29755973815918, 4.382428169250488, 4.292300224304199, 4.571993827819824, 4.142297267913818, 4.494933128356934, 4.165191173553467, 4.180934906005859, 4.4890336990356445, 4.077141761779785, 4.032059669494629, 4.236023902893066, 4.219968795776367, 4.233877182006836, 4.190492153167725, 4.176075458526611, 4.234570503234863, 3.9322190284729004, 4.118359565734863, 4.1095075607299805, 4.117043495178223, 4.384971618652344, 4.160821914672852, 3.943291664123535, 4.368199348449707, 4.237608432769775, 4.116882801055908, 3.99635910987854, 4.293118476867676, 4.2008771896362305, 4.151986598968506, 3.9951884746551514, 4.171354293823242, 3.799830436706543, 4.037522792816162, 4.114823341369629, 4.157523155212402, 4.025557518005371, 4.240346908569336, 3.9659337997436523, 3.950904369354248, 4.01806640625, 4.100918292999268, 4.172019958496094, 4.1105451583862305, 4.008934497833252, 3.9844353199005127, 4.079896450042725, 3.9156720638275146, 4.020744323730469, 4.096451759338379, 4.108036518096924, 4.019435882568359], "val_loss": [5.676942825317383, 5.0648016929626465, 4.498353958129883, 4.205862522125244, 4.01993989944458], "val_pp": [292.0540847041283, 158.34849776052857, 89.86880928696887, 67.07823958130747, 55.69760765215066], "val_acc": [0.18198013305664062, 0.22062301635742188, 0.2741413116455078, 0.2999076843261719, 0.31641578674316406], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.676942825317383, "val_perplexity": 292.0540847041283, "val_accuracy": 0.18198013305664062, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 5.0648016929626465, "val_perplexity": 158.34849776052857, "val_accuracy": 0.22062301635742188, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.498353958129883, "val_perplexity": 89.86880928696887, "val_accuracy": 0.2741413116455078, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.205862522125244, "val_perplexity": 67.07823958130747, "val_accuracy": 0.2999076843261719, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.01993989944458, "val_perplexity": 55.69760765215066, "val_accuracy": 0.31641578674316406, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.272572634, 0.274153173, 0.275549448, 0.274169102, 0.275614155, 0.273366953, 0.274622258, 0.27380532, 0.274341766, 0.277965882, 0.274131338, 0.275233957, 0.275714927, 0.276184847, 0.274198599, 0.27396199, 0.275568096, 0.274085069, 0.274452188, 0.275256797, 0.27522648, 0.273885084, 0.275619953, 0.275417946, 0.274423066, 0.275091113, 0.274471805, 0.274785962, 0.275821021, 0.274993715, 0.278092981, 0.273484519, 0.275556947, 0.274209963, 0.27540259, 0.274996741, 0.276256274, 0.275418725, 0.274247946, 0.273978239, 0.274696747, 0.2743578, 0.275269625, 0.276610397, 0.276123268, 0.274492073, 0.274247031, 0.275440518, 0.274270096, 0.275186853, 0.275971084, 0.274552085, 0.275952188, 0.274617493, 0.275718214, 0.275791701, 0.274436647, 0.278252973, 0.273984712, 0.275120651, 0.276086819, 0.276379089, 0.274536728, 0.274607638, 0.27640672, 0.275079779, 0.275453987, 0.276191085, 0.274076302, 0.274565042, 0.275100048, 0.279121993, 0.276544291, 0.276241127, 0.275569698, 0.274652585, 0.274266567, 0.277462492, 0.275583954, 0.274431855, 0.273970571], "train_time_seconds": 1127.8803733319994, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 237999.93540714282, "peak_memory_bytes": 6314994176, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "ademamix", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.999, "adema_alpha": 8.0, "adema_beta3_warmup": 4096, "adema_alpha_warmup": 4096, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456/run.log new file mode 100644 index 0000000..dfdc96b --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456/run.log @@ -0,0 +1,143 @@ +Starting Experiment: sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adopt', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +ADOPT ( +Parameter Group 0 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.0003 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.0003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.382 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.561 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.227 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.155 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=5.967 iter_dt=2.74e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.867 val_pp=353.188 val_acc=0.180328 +Train: Iter=300 (0.036622 effective passes) train_loss=5.785 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=350 (0.042725 effective passes) train_loss=5.839 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=400 (0.048829 effective passes) train_loss=5.533 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=450 (0.054932 effective passes) train_loss=5.332 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=500 (0.061036 effective passes) train_loss=5.274 iter_dt=2.75e-01s lr=3.00e-04 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.325 val_pp=205.400 val_acc=0.211224 +Train: Iter=550 (0.067140 effective passes) train_loss=5.448 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=600 (0.073243 effective passes) train_loss=4.917 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=650 (0.079347 effective passes) train_loss=5.399 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=700 (0.085451 effective passes) train_loss=5.181 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=750 (0.091554 effective passes) train_loss=5.128 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=800 (0.097658 effective passes) train_loss=4.949 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=850 (0.103761 effective passes) train_loss=4.936 iter_dt=2.77e-01s lr=3.00e-04 +Train: Iter=900 (0.109865 effective passes) train_loss=4.869 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=950 (0.115969 effective passes) train_loss=4.884 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.860 iter_dt=2.74e-01s lr=3.00e-04 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.819 val_pp=123.854 val_acc=0.247356 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.908 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.737 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.682 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.625 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.635 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.701 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.667 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.649 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.524 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.645 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.512 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.821 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.368 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.720 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.374 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.410 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.700 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.301 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.281 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.447 iter_dt=2.74e-01s lr=3.00e-04 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.414 val_pp=82.565 val_acc=0.283892 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.461 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.419 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.397 iter_dt=2.76e-01s lr=3.00e-04 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.354 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.427 iter_dt=2.78e-01s lr=3.00e-04 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.153 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.326 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.339 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.314 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.561 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.372 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.118 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.519 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.418 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.312 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.170 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.465 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.384 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.301 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.154 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.346 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.928 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.215 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.271 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.344 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.203 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.400 iter_dt=2.80e-01s lr=3.00e-04 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.118 iter_dt=2.92e-01s lr=3.00e-04 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.076 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.177 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.256 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.327 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.289 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.140 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.148 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.268 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.073 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.163 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.219 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.268 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.154 iter_dt=2.73e-01s lr=3.00e-04 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.171 val_pp=64.765 val_acc=0.303391 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456/summary.json new file mode 100644 index 0000000..d282c3a --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.381633281707764, 6.560956001281738, 6.227297782897949, 6.155085563659668, 5.967130184173584, 5.784832954406738, 5.838903427124023, 5.533449172973633, 5.332378387451172, 5.274155616760254, 5.448360443115234, 4.917234420776367, 5.398746013641357, 5.181194305419922, 5.128459930419922, 4.948990821838379, 4.935697555541992, 4.868923187255859, 4.884421348571777, 4.860377311706543, 4.907842636108398, 4.7371697425842285, 4.681623458862305, 4.625258445739746, 4.634853363037109, 4.701051712036133, 4.667119026184082, 4.6493754386901855, 4.523929595947266, 4.644521713256836, 4.5124359130859375, 4.821046829223633, 4.368030548095703, 4.719653129577637, 4.374360084533691, 4.409933090209961, 4.699725151062012, 4.300860404968262, 4.281021595001221, 4.446754455566406, 4.461247444152832, 4.419081687927246, 4.397184371948242, 4.3537187576293945, 4.427000045776367, 4.152863502502441, 4.325507640838623, 4.3390374183654785, 4.314126491546631, 4.5608906745910645, 4.372016906738281, 4.118013858795166, 4.519289016723633, 4.418260097503662, 4.312141418457031, 4.170322895050049, 4.464642524719238, 4.383833885192871, 4.301037311553955, 4.153754711151123, 4.3460235595703125, 3.9276785850524902, 4.214939117431641, 4.271388053894043, 4.343825340270996, 4.203242778778076, 4.399658679962158, 4.117999076843262, 4.075884819030762, 4.177437782287598, 4.255593299865723, 4.327383518218994, 4.289126873016357, 4.139743328094482, 4.148109436035156, 4.26767635345459, 4.072595596313477, 4.163387775421143, 4.219487190246582, 4.2679619789123535, 4.154379367828369], "val_loss": [5.867005348205566, 5.324961185455322, 4.819103240966797, 4.413585662841797, 4.170770168304443], "val_pp": [353.18832093529016, 205.3996520755454, 123.85357231261972, 82.56473844410924, 64.76513139810707], "val_acc": [0.180328369140625, 0.21122360229492188, 0.24735641479492188, 0.2838916778564453, 0.3033905029296875], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.867005348205566, "val_perplexity": 353.18832093529016, "val_accuracy": 0.180328369140625, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 5.324961185455322, "val_perplexity": 205.3996520755454, "val_accuracy": 0.21122360229492188, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.819103240966797, "val_perplexity": 123.85357231261972, "val_accuracy": 0.24735641479492188, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.413585662841797, "val_perplexity": 82.56473844410924, "val_accuracy": 0.2838916778564453, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.170770168304443, "val_perplexity": 64.76513139810707, "val_accuracy": 0.3033905029296875, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.273260081, 0.274172196, 0.27476981, 0.27411463, 0.273717887, 0.273295597, 0.272989279, 0.273647412, 0.273852445, 0.275171458, 0.27539082, 0.273757106, 0.273630391, 0.273798721, 0.273536073, 0.274258574, 0.276516823, 0.273811325, 0.273836027, 0.274210981, 0.273832526, 0.274077353, 0.273547756, 0.27402196, 0.274189399, 0.273413341, 0.274984242, 0.273511371, 0.273542973, 0.273329137, 0.274960725, 0.273986838, 0.273426182, 0.274298787, 0.273408901, 0.273294016, 0.273304827, 0.273160596, 0.273844223, 0.273875345, 0.273945215, 0.274211924, 0.275781571, 0.275043153, 0.277632202, 0.274003055, 0.273460313, 0.274007422, 0.274185986, 0.273803424, 0.274769247, 0.273460058, 0.273434388, 0.274142088, 0.274580262, 0.273408861, 0.274411012, 0.275140012, 0.273164184, 0.273352767, 0.273046939, 0.273257995, 0.273308349, 0.273179629, 0.274290173, 0.274679932, 0.280139432, 0.292302524, 0.274740387, 0.274276094, 0.274027017, 0.274292054, 0.27354433, 0.27397901, 0.273805447, 0.274798611, 0.273473708, 0.273806035, 0.274552271, 0.273347415, 0.273338347], "train_time_seconds": 1122.9521831029995, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 239044.4224065225, "peak_memory_bytes": 6175400960, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "adopt", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456/run.log new file mode 100644 index 0000000..3714183 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456/run.log @@ -0,0 +1,150 @@ +Starting Experiment: sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': 'nccl', 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'd-muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Distributed DataReader Initialized for Worker 0/1 +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +DistributedMuon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.003 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.1 + +Parameter Group 1 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.003 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.593 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=5.888 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=5.545 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.357 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.291 iter_dt=2.81e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.120 val_pp=167.299 val_acc=0.216290 +Train: Iter=300 (0.036622 effective passes) train_loss=4.916 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.066 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=4.672 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=4.495 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=4.534 iter_dt=2.80e-01s lr=3.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=4.620 val_pp=101.444 val_acc=0.263233 +Train: Iter=550 (0.067140 effective passes) train_loss=4.581 iter_dt=2.83e-01s lr=3.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.359 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=4.739 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.622 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.613 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.484 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.389 iter_dt=2.83e-01s lr=3.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.356 iter_dt=2.83e-01s lr=3.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.454 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.481 iter_dt=2.81e-01s lr=3.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.364 val_pp=78.575 val_acc=0.284447 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.503 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.313 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.329 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.257 iter_dt=2.83e-01s lr=3.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.288 iter_dt=2.85e-01s lr=3.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.339 iter_dt=2.84e-01s lr=3.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.355 iter_dt=2.88e-01s lr=3.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.381 iter_dt=2.83e-01s lr=3.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.247 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.321 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.259 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.524 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.103 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.459 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.129 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.163 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.475 iter_dt=3.00e-01s lr=3.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.055 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.012 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.220 iter_dt=2.81e-01s lr=3.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.190 val_pp=65.992 val_acc=0.298725 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.184 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.216 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.191 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.186 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.213 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.925 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.117 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.113 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.109 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.376 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.151 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.948 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.361 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.248 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.136 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.993 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.300 iter_dt=2.83e-01s lr=3.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.215 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.164 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.018 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.192 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.819 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.039 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.129 iter_dt=2.84e-01s lr=3.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.188 iter_dt=2.91e-01s lr=3.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.056 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.270 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.013 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.964 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.054 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.127 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.227 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.152 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.040 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.023 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.122 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.961 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.064 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.123 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.151 iter_dt=2.81e-01s lr=3.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.062 iter_dt=2.82e-01s lr=3.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.057 val_pp=57.809 val_acc=0.309958 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456/summary.json new file mode 100644 index 0000000..ad00de2 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [6.592528820037842, 5.888398170471191, 5.5451154708862305, 5.357484817504883, 5.290867805480957, 4.916447162628174, 5.065686225891113, 4.6715850830078125, 4.4953508377075195, 4.534110069274902, 4.580676078796387, 4.358577728271484, 4.738816261291504, 4.621905326843262, 4.612517356872559, 4.48389196395874, 4.389326572418213, 4.356166839599609, 4.453569412231445, 4.481386184692383, 4.5027899742126465, 4.313022613525391, 4.329314708709717, 4.257477760314941, 4.288460731506348, 4.339213848114014, 4.354570388793945, 4.381494522094727, 4.2471771240234375, 4.320516586303711, 4.258951663970947, 4.523875713348389, 4.102516174316406, 4.458558082580566, 4.1291632652282715, 4.163052558898926, 4.475366115570068, 4.054927349090576, 4.01155424118042, 4.2196478843688965, 4.184042453765869, 4.216005802154541, 4.191255569458008, 4.186071872711182, 4.213373184204102, 3.9254021644592285, 4.116702079772949, 4.112605094909668, 4.108559608459473, 4.375631332397461, 4.151391983032227, 3.9477005004882812, 4.36061954498291, 4.247880458831787, 4.136335372924805, 3.9931998252868652, 4.3003435134887695, 4.214511871337891, 4.163858890533447, 4.0176591873168945, 4.191699981689453, 3.818796157836914, 4.039051532745361, 4.128955841064453, 4.187841415405273, 4.0556254386901855, 4.270148277282715, 4.013184547424316, 3.9640378952026367, 4.053966522216797, 4.127445220947266, 4.226717948913574, 4.1519365310668945, 4.039616107940674, 4.0233845710754395, 4.121722221374512, 3.9611196517944336, 4.064257621765137, 4.123442649841309, 4.150893211364746, 4.061779022216797], "val_loss": [5.119787216186523, 4.61951208114624, 4.364060401916504, 4.189532279968262, 4.057147979736328], "val_pp": [167.2991909985406, 101.4442081359866, 78.57530516513165, 65.99173202746029, 57.80904496300566], "val_acc": [0.21628952026367188, 0.2632331848144531, 0.28444671630859375, 0.2987251281738281, 0.30995750427246094], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.119787216186523, "val_perplexity": 167.2991909985406, "val_accuracy": 0.21628952026367188, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 4.61951208114624, "val_perplexity": 101.4442081359866, "val_accuracy": 0.2632331848144531, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.364060401916504, "val_perplexity": 78.57530516513165, "val_accuracy": 0.28444671630859375, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.189532279968262, "val_perplexity": 65.99173202746029, "val_accuracy": 0.2987251281738281, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.057147979736328, "val_perplexity": 57.80904496300566, "val_accuracy": 0.30995750427246094, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.282173591, 0.281190565, 0.281324232, 0.281497996, 0.281419305, 0.280305697, 0.281073442, 0.280421565, 0.280438097, 0.280240647, 0.282952266, 0.281537929, 0.282307789, 0.280743675, 0.282039649, 0.282106695, 0.283208581, 0.283105359, 0.281219113, 0.280963751, 0.281146661, 0.281427947, 0.281786871, 0.283166083, 0.284549745, 0.283633197, 0.287866614, 0.282597391, 0.280650297, 0.27999911, 0.280556471, 0.280323749, 0.279739312, 0.28012667, 0.27999386, 0.280372564, 0.299798202, 0.281314117, 0.28095541, 0.280565628, 0.281346974, 0.279818029, 0.279827973, 0.280682133, 0.281562342, 0.281859663, 0.282320613, 0.280685485, 0.280407269, 0.280885055, 0.281996677, 0.281010643, 0.281983627, 0.280665297, 0.280229103, 0.280600279, 0.282579381, 0.281615721, 0.281450459, 0.282391856, 0.281339269, 0.280952969, 0.280211909, 0.283660162, 0.290543415, 0.280527133, 0.280754789, 0.280633223, 0.280004724, 0.280842369, 0.280914567, 0.280506585, 0.280495625, 0.280231175, 0.280418861, 0.281306009, 0.282132942, 0.280897383, 0.280404229, 0.280669299, 0.281608498], "train_time_seconds": 1153.460835318, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 232721.77761110934, "peak_memory_bytes": 6252898816, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": "nccl", "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "d-muon", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.95, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": true, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456/run.log new file mode 100644 index 0000000..6bbf218 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456/run.log @@ -0,0 +1,141 @@ +Starting Experiment: sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.027, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lamb', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lamb ( +Parameter Group 0 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.027 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.027 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.723 iter_dt=2.80e-01s lr=2.70e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=6.794 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=6.423 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=6.328 iter_dt=2.85e-01s lr=2.70e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=6.140 iter_dt=2.81e-01s lr=2.70e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.028 val_pp=415.060 val_acc=0.168570 +Train: Iter=300 (0.036622 effective passes) train_loss=5.780 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=350 (0.042725 effective passes) train_loss=5.951 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=400 (0.048829 effective passes) train_loss=5.663 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=450 (0.054932 effective passes) train_loss=5.432 iter_dt=2.86e-01s lr=2.70e-02 +Train: Iter=500 (0.061036 effective passes) train_loss=5.352 iter_dt=2.83e-01s lr=2.70e-02 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.411 val_pp=223.751 val_acc=0.202606 +Train: Iter=550 (0.067140 effective passes) train_loss=5.519 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=600 (0.073243 effective passes) train_loss=4.987 iter_dt=2.84e-01s lr=2.70e-02 +Train: Iter=650 (0.079347 effective passes) train_loss=5.454 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=700 (0.085451 effective passes) train_loss=5.258 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=750 (0.091554 effective passes) train_loss=5.189 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=800 (0.097658 effective passes) train_loss=5.003 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=850 (0.103761 effective passes) train_loss=4.954 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=900 (0.109865 effective passes) train_loss=4.879 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=950 (0.115969 effective passes) train_loss=4.913 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.888 iter_dt=2.82e-01s lr=2.70e-02 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.842 val_pp=126.748 val_acc=0.246984 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.933 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.734 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.734 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.671 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.673 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.707 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.698 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.699 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.577 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.650 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.549 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.862 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.410 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.754 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.425 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.452 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.752 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.327 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.333 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.491 iter_dt=2.81e-01s lr=2.70e-02 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.456 val_pp=86.124 val_acc=0.279652 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.493 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.463 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.430 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.404 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.464 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.173 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.350 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.361 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.339 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.613 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.400 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.159 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.566 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.448 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.344 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.200 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.498 iter_dt=2.85e-01s lr=2.70e-02 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.428 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.357 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.209 iter_dt=2.88e-01s lr=2.70e-02 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.372 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.992 iter_dt=2.84e-01s lr=2.70e-02 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.250 iter_dt=2.84e-01s lr=2.70e-02 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.318 iter_dt=2.86e-01s lr=2.70e-02 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.374 iter_dt=2.87e-01s lr=2.70e-02 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.234 iter_dt=2.87e-01s lr=2.70e-02 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.441 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.182 iter_dt=2.87e-01s lr=2.70e-02 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.130 iter_dt=2.84e-01s lr=2.70e-02 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.239 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.325 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.380 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.334 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.181 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.202 iter_dt=2.83e-01s lr=2.70e-02 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.306 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.122 iter_dt=2.81e-01s lr=2.70e-02 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.215 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.268 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.328 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.197 iter_dt=2.84e-01s lr=2.70e-02 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.211 val_pp=67.393 val_acc=0.298441 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456/summary.json new file mode 100644 index 0000000..3fe99ea --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.722756862640381, 6.794173240661621, 6.422684669494629, 6.328466415405273, 6.140127182006836, 5.779804706573486, 5.951230525970459, 5.662991046905518, 5.432400703430176, 5.35192346572876, 5.519402980804443, 4.986542701721191, 5.453757286071777, 5.257530212402344, 5.188511848449707, 5.003031253814697, 4.953614234924316, 4.879005432128906, 4.913366317749023, 4.888352394104004, 4.933265686035156, 4.733579635620117, 4.733677387237549, 4.671250343322754, 4.673291206359863, 4.707294940948486, 4.6982598304748535, 4.699019432067871, 4.577014446258545, 4.649850368499756, 4.5490617752075195, 4.862076282501221, 4.410113334655762, 4.754430294036865, 4.424936294555664, 4.451800346374512, 4.751544952392578, 4.32652473449707, 4.3326263427734375, 4.490525245666504, 4.493192672729492, 4.463449478149414, 4.430263996124268, 4.403997421264648, 4.463874816894531, 4.1732025146484375, 4.350398540496826, 4.360682487487793, 4.338603496551514, 4.612649917602539, 4.399710178375244, 4.158685684204102, 4.566433906555176, 4.44818115234375, 4.34370756149292, 4.1996917724609375, 4.498492240905762, 4.427672863006592, 4.356873989105225, 4.209436416625977, 4.372453689575195, 3.9917593002319336, 4.2497663497924805, 4.317758083343506, 4.3739213943481445, 4.234414100646973, 4.441074848175049, 4.1816205978393555, 4.129834175109863, 4.239090919494629, 4.324566841125488, 4.379935264587402, 4.333680629730225, 4.18099308013916, 4.201676368713379, 4.306367874145508, 4.122293472290039, 4.214663505554199, 4.268488883972168, 4.3276166915893555, 4.197102069854736], "val_loss": [6.028428077697754, 5.410538196563721, 4.8422064781188965, 4.455793380737305, 4.210540771484375], "val_pp": [415.06038790102406, 223.75116350202543, 126.74829883936543, 86.1241950969045, 67.3927832843249], "val_acc": [0.16856956481933594, 0.202606201171875, 0.24698448181152344, 0.2796516418457031, 0.29844093322753906], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.028428077697754, "val_perplexity": 415.06038790102406, "val_accuracy": 0.16856956481933594, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 5.410538196563721, "val_perplexity": 223.75116350202543, "val_accuracy": 0.202606201171875, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.8422064781188965, "val_perplexity": 126.74829883936543, "val_accuracy": 0.24698448181152344, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.455793380737305, "val_perplexity": 86.1241950969045, "val_accuracy": 0.2796516418457031, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.210540771484375, "val_perplexity": 67.3927832843249, "val_accuracy": 0.29844093322753906, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.280035852, 0.282557025, 0.28284384, 0.285393529, 0.280782672, 0.282253586, 0.283399314, 0.282289592, 0.285846921, 0.28265425, 0.283027374, 0.283555246, 0.280932993, 0.281275213, 0.282850472, 0.282469989, 0.282556791, 0.281965649, 0.28220225, 0.282409745, 0.281952688, 0.282641351, 0.281303994, 0.281111938, 0.281825509, 0.282290842, 0.282684373, 0.283319041, 0.281973583, 0.281904582, 0.281313702, 0.281855802, 0.28136775, 0.28134272, 0.281969294, 0.28250712, 0.282081643, 0.282650137, 0.282771024, 0.281329885, 0.282549694, 0.281771946, 0.28187053, 0.28174359, 0.281848203, 0.28148835, 0.282158407, 0.281395237, 0.283256757, 0.281456119, 0.282169025, 0.28219931, 0.28118774, 0.281226529, 0.282946042, 0.282359954, 0.28484954, 0.2832242, 0.283093445, 0.287662702, 0.283171472, 0.28412504, 0.284321334, 0.285660551, 0.287063646, 0.286838954, 0.282993538, 0.286701909, 0.284224185, 0.281047072, 0.281887729, 0.281841321, 0.28066951, 0.281176092, 0.282636963, 0.282028382, 0.281466013, 0.281785983, 0.281889915, 0.282417193, 0.283669447], "train_time_seconds": 1158.5195744759978, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 231705.5852262265, "peak_memory_bytes": 6175793152, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.027, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "lamb", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456/run.log new file mode 100644 index 0000000..83a695f --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456/run.log @@ -0,0 +1,139 @@ +Starting Experiment: sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.99, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lion ( +Parameter Group 0 + betas: (0.9, 0.99) + lr: 0.0003 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.99) + lr: 0.0003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.616 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.890 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.661 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.575 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.403 iter_dt=2.73e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.305 val_pp=547.424 val_acc=0.142797 +Train: Iter=300 (0.036622 effective passes) train_loss=6.054 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=350 (0.042725 effective passes) train_loss=6.198 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=400 (0.048829 effective passes) train_loss=5.938 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=450 (0.054932 effective passes) train_loss=5.725 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=500 (0.061036 effective passes) train_loss=5.603 iter_dt=2.75e-01s lr=3.00e-04 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.674 val_pp=291.197 val_acc=0.180149 +Train: Iter=550 (0.067140 effective passes) train_loss=5.809 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=600 (0.073243 effective passes) train_loss=5.239 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=650 (0.079347 effective passes) train_loss=5.693 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=700 (0.085451 effective passes) train_loss=5.442 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=750 (0.091554 effective passes) train_loss=5.414 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=800 (0.097658 effective passes) train_loss=5.194 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=850 (0.103761 effective passes) train_loss=5.174 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=900 (0.109865 effective passes) train_loss=5.087 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=950 (0.115969 effective passes) train_loss=5.102 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=1000 (0.122072 effective passes) train_loss=5.069 iter_dt=2.72e-01s lr=3.00e-04 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=5.056 val_pp=156.948 val_acc=0.222181 +Train: Iter=1050 (0.128176 effective passes) train_loss=5.153 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.953 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.881 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.850 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.825 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.920 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.827 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.820 iter_dt=2.76e-01s lr=3.00e-04 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.693 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.757 iter_dt=2.76e-01s lr=3.00e-04 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.634 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.917 iter_dt=2.79e-01s lr=3.00e-04 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.470 iter_dt=2.77e-01s lr=3.00e-04 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.802 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.485 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.494 iter_dt=2.76e-01s lr=3.00e-04 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.769 iter_dt=2.76e-01s lr=3.00e-04 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.373 iter_dt=2.76e-01s lr=3.00e-04 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.346 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.516 iter_dt=2.75e-01s lr=3.00e-04 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.471 val_pp=87.436 val_acc=0.277426 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.485 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.489 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.436 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.408 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.445 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.174 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.333 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.367 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.345 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.595 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.372 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.124 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.555 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.435 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.323 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.166 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.471 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.407 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.317 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.164 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.344 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.959 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.222 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.278 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.347 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.213 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.414 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.139 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.085 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.179 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.261 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.338 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.282 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.128 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.142 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.255 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.052 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.155 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.216 iter_dt=2.78e-01s lr=3.00e-04 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.272 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.159 iter_dt=2.72e-01s lr=3.00e-04 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.167 val_pp=64.520 val_acc=0.303646 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456/summary.json new file mode 100644 index 0000000..9b747d5 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.616348743438721, 6.890268325805664, 6.660611152648926, 6.574917793273926, 6.403221607208252, 6.054110527038574, 6.19774055480957, 5.937782287597656, 5.724968910217285, 5.602939605712891, 5.809439659118652, 5.239278793334961, 5.692577838897705, 5.44215202331543, 5.414319038391113, 5.193840026855469, 5.173692226409912, 5.086586952209473, 5.102476119995117, 5.068856239318848, 5.152618408203125, 4.953420162200928, 4.880972862243652, 4.850339889526367, 4.824958324432373, 4.920385360717773, 4.826552867889404, 4.81976842880249, 4.692680358886719, 4.757078170776367, 4.6342339515686035, 4.916977882385254, 4.4696125984191895, 4.802184581756592, 4.485002517700195, 4.4942779541015625, 4.769254684448242, 4.373433589935303, 4.34621000289917, 4.515668869018555, 4.484545707702637, 4.4889936447143555, 4.435659885406494, 4.407618522644043, 4.445013999938965, 4.1743621826171875, 4.333170413970947, 4.367181777954102, 4.344826698303223, 4.594962120056152, 4.371940612792969, 4.124458312988281, 4.555466175079346, 4.435121536254883, 4.322571754455566, 4.165506839752197, 4.47058629989624, 4.407474517822266, 4.317007064819336, 4.163705348968506, 4.3439836502075195, 3.9586305618286133, 4.222439765930176, 4.278085708618164, 4.346518039703369, 4.213461875915527, 4.413715362548828, 4.138999938964844, 4.08542537689209, 4.179429531097412, 4.261317253112793, 4.338055610656738, 4.282296657562256, 4.1281538009643555, 4.142086982727051, 4.255480766296387, 4.05171012878418, 4.15462589263916, 4.215757369995117, 4.2718915939331055, 4.158726692199707], "val_loss": [6.305228233337402, 5.674002647399902, 5.055916786193848, 4.4709062576293945, 4.166975021362305], "val_pp": [547.4241931523586, 291.19665542532005, 156.94781872872173, 87.43566364260828, 64.51980419359604], "val_acc": [0.14279747009277344, 0.18014907836914062, 0.2221813201904297, 0.27742576599121094, 0.3036460876464844], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.305228233337402, "val_perplexity": 547.4241931523586, "val_accuracy": 0.14279747009277344, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 5.674002647399902, "val_perplexity": 291.19665542532005, "val_accuracy": 0.18014907836914062, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 5.055916786193848, "val_perplexity": 156.94781872872173, "val_accuracy": 0.2221813201904297, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.4709062576293945, "val_perplexity": 87.43566364260828, "val_accuracy": 0.27742576599121094, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.166975021362305, "val_perplexity": 64.51980419359604, "val_accuracy": 0.3036460876464844, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.271905653, 0.273772893, 0.272521901, 0.272499605, 0.272877736, 0.272735169, 0.272754675, 0.273084459, 0.273365604, 0.274685912, 0.272460821, 0.272492748, 0.273029141, 0.272842615, 0.272467215, 0.272654948, 0.272661763, 0.272679544, 0.2723249, 0.272159018, 0.273233778, 0.27279345, 0.273563114, 0.273477233, 0.271837079, 0.272090359, 0.272117798, 0.275634933, 0.274097492, 0.275528229, 0.273069734, 0.279111487, 0.276593685, 0.273408966, 0.274228034, 0.275949611, 0.275526395, 0.275810978, 0.274878646, 0.274728407, 0.271787802, 0.270781222, 0.271196291, 0.271435567, 0.271760076, 0.271740578, 0.271925743, 0.273495741, 0.272124986, 0.271531065, 0.271566454, 0.270512669, 0.271298517, 0.271837755, 0.27224225, 0.271458202, 0.272799439, 0.272627118, 0.270843185, 0.272236738, 0.272237401, 0.271489938, 0.272836432, 0.274940066, 0.271328028, 0.27150294, 0.27280677, 0.272802011, 0.272921001, 0.272985028, 0.272776368, 0.272489046, 0.271741371, 0.271657284, 0.271810883, 0.27441467, 0.27188884, 0.274242486, 0.277843408, 0.271898025, 0.272205907], "train_time_seconds": 1117.527238096001, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 240204.84409610435, "peak_memory_bytes": 6039584256, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "lion", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.99, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456/run.log new file mode 100644 index 0000000..e8d779c --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456/run.log @@ -0,0 +1,151 @@ +Starting Experiment: sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'mars', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.0009, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +MARS ( +Parameter Group 0 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.0009 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.1 + weight_decay_1d: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.0009 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.0 + weight_decay_1d: 0.1 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.252 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.459 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.181 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.120 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=5.951 iter_dt=2.79e-01s lr=9.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.858 val_pp=349.918 val_acc=0.177553 +Train: Iter=300 (0.036622 effective passes) train_loss=5.710 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=350 (0.042725 effective passes) train_loss=5.803 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=400 (0.048829 effective passes) train_loss=5.515 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=450 (0.054932 effective passes) train_loss=5.306 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=500 (0.061036 effective passes) train_loss=5.255 iter_dt=2.81e-01s lr=9.00e-04 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.305 val_pp=201.331 val_acc=0.208384 +Train: Iter=550 (0.067140 effective passes) train_loss=5.417 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=600 (0.073243 effective passes) train_loss=4.906 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=650 (0.079347 effective passes) train_loss=5.362 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=700 (0.085451 effective passes) train_loss=5.145 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=750 (0.091554 effective passes) train_loss=5.087 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=800 (0.097658 effective passes) train_loss=4.916 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=850 (0.103761 effective passes) train_loss=4.882 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=900 (0.109865 effective passes) train_loss=4.808 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=950 (0.115969 effective passes) train_loss=4.826 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.817 iter_dt=2.80e-01s lr=9.00e-04 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.751 val_pp=115.662 val_acc=0.252459 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.843 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.664 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.633 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.570 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.590 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.631 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.611 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.607 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.484 iter_dt=2.83e-01s lr=9.00e-04 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.576 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.467 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.756 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.300 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.657 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.323 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.350 iter_dt=2.83e-01s lr=9.00e-04 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.645 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.236 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.222 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.392 iter_dt=2.94e-01s lr=9.00e-04 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.355 val_pp=77.860 val_acc=0.287682 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.407 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.368 iter_dt=2.83e-01s lr=9.00e-04 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.324 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.293 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.356 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.091 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.255 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.259 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.263 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.504 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.298 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.053 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.487 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.355 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.262 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.105 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.413 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.324 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.256 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.103 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.289 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.896 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.141 iter_dt=2.84e-01s lr=9.00e-04 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.210 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.270 iter_dt=2.87e-01s lr=9.00e-04 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.155 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.369 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.080 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.049 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.136 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.222 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.288 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.240 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.094 iter_dt=2.80e-01s lr=9.00e-04 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.115 iter_dt=2.79e-01s lr=9.00e-04 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.202 iter_dt=2.83e-01s lr=9.00e-04 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.025 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.113 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.172 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.230 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.095 iter_dt=2.83e-01s lr=9.00e-04 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.122 val_pp=61.670 val_acc=0.305550 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456/summary.json new file mode 100644 index 0000000..842a818 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.252167224884033, 6.459163188934326, 6.1809587478637695, 6.119510173797607, 5.95059061050415, 5.710448265075684, 5.802942276000977, 5.515110015869141, 5.306466579437256, 5.2552289962768555, 5.417483329772949, 4.906240940093994, 5.362061500549316, 5.145374298095703, 5.086697578430176, 4.916474342346191, 4.881856441497803, 4.807552337646484, 4.826048851013184, 4.816788196563721, 4.842556953430176, 4.66447639465332, 4.633466720581055, 4.5703582763671875, 4.590449810028076, 4.630958080291748, 4.61087703704834, 4.607010841369629, 4.483617782592773, 4.575662612915039, 4.467218399047852, 4.7560224533081055, 4.300301551818848, 4.657437324523926, 4.323246002197266, 4.350205421447754, 4.645419120788574, 4.235630989074707, 4.221959114074707, 4.392441749572754, 4.4072418212890625, 4.3677568435668945, 4.324275970458984, 4.293409824371338, 4.356157302856445, 4.090503692626953, 4.255240440368652, 4.259011268615723, 4.262894153594971, 4.503781795501709, 4.297743797302246, 4.052539825439453, 4.48671817779541, 4.354951858520508, 4.261935234069824, 4.10487174987793, 4.413445472717285, 4.324241638183594, 4.256214141845703, 4.10264778137207, 4.289043426513672, 3.896073818206787, 4.140673637390137, 4.210172653198242, 4.270430088043213, 4.155110836029053, 4.368578910827637, 4.080420970916748, 4.049270153045654, 4.135547637939453, 4.221693992614746, 4.288382053375244, 4.240026473999023, 4.093845844268799, 4.11512565612793, 4.202199935913086, 4.02537202835083, 4.113348960876465, 4.171680450439453, 4.2303786277771, 4.094624996185303], "val_loss": [5.857702732086182, 5.304955005645752, 4.750679016113281, 4.3549089431762695, 4.1218061447143555], "val_pp": [349.91798270233926, 201.3312249790169, 115.66242516418683, 77.85950727643227, 61.670356648930785], "val_acc": [0.1775531768798828, 0.20838356018066406, 0.2524585723876953, 0.28768157958984375, 0.30554962158203125], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.857702732086182, "val_perplexity": 349.91798270233926, "val_accuracy": 0.1775531768798828, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 5.304955005645752, "val_perplexity": 201.3312249790169, "val_accuracy": 0.20838356018066406, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.750679016113281, "val_perplexity": 115.66242516418683, "val_accuracy": 0.2524585723876953, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.3549089431762695, "val_perplexity": 77.85950727643227, "val_accuracy": 0.28768157958984375, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.1218061447143555, "val_perplexity": 61.670356648930785, "val_accuracy": 0.30554962158203125, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.278597033, 0.279236674, 0.280256255, 0.279103386, 0.279359495, 0.280060227, 0.280619307, 0.281192041, 0.280444405, 0.281353713, 0.28100426, 0.280737307, 0.280080763, 0.280838166, 0.278883265, 0.281800253, 0.280228115, 0.280972922, 0.279952474, 0.279903011, 0.279569861, 0.27956757, 0.280829977, 0.280139626, 0.280444203, 0.280041349, 0.279995374, 0.279536153, 0.282675965, 0.280438276, 0.280697584, 0.280478564, 0.281080416, 0.28051065, 0.280879036, 0.283080826, 0.282174152, 0.282395634, 0.281372661, 0.293949255, 0.281596197, 0.28281653, 0.281327497, 0.281260296, 0.281861274, 0.282160925, 0.282201914, 0.279703407, 0.281197377, 0.279604766, 0.279666824, 0.279672897, 0.280288206, 0.280416493, 0.280482798, 0.280448252, 0.281294999, 0.280290727, 0.281438716, 0.280810468, 0.280576314, 0.280442283, 0.283638448, 0.2799109, 0.287261421, 0.279487004, 0.279748146, 0.279447679, 0.281821648, 0.280346176, 0.279348455, 0.2791752, 0.279148984, 0.279588976, 0.279407912, 0.282920494, 0.280914453, 0.280580681, 0.281533763, 0.28113259, 0.282800513], "train_time_seconds": 1150.9215605589964, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 233235.23096536862, "peak_memory_bytes": 6324693504, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "mars", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.0009, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456/run.log new file mode 100644 index 0000000..b596288 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456/run.log @@ -0,0 +1,129 @@ +Starting Experiment: sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.06, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon-pytorch', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.003, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: + +Train: Iter=50 (0.006104 effective passes) train_loss=6.777 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=5.844 iter_dt=2.83e-01s lr=6.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=5.551 iter_dt=2.85e-01s lr=6.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=5.476 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=5.435 iter_dt=2.82e-01s lr=6.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.278 val_pp=195.998 val_acc=0.204409 +Train: Iter=300 (0.036622 effective passes) train_loss=5.093 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=350 (0.042725 effective passes) train_loss=5.293 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=400 (0.048829 effective passes) train_loss=4.968 iter_dt=2.85e-01s lr=6.00e-02 +Train: Iter=450 (0.054932 effective passes) train_loss=4.828 iter_dt=2.81e-01s lr=6.00e-02 +Train: Iter=500 (0.061036 effective passes) train_loss=4.875 iter_dt=2.88e-01s lr=6.00e-02 +>Eval: Iter=512 (0.062501 effective passes) val_loss=4.949 val_pp=141.029 val_acc=0.237040 +Train: Iter=550 (0.067140 effective passes) train_loss=4.966 iter_dt=2.86e-01s lr=6.00e-02 +Train: Iter=600 (0.073243 effective passes) train_loss=4.692 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=650 (0.079347 effective passes) train_loss=5.086 iter_dt=2.84e-01s lr=6.00e-02 +Train: Iter=700 (0.085451 effective passes) train_loss=4.971 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=750 (0.091554 effective passes) train_loss=5.011 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=800 (0.097658 effective passes) train_loss=4.838 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=850 (0.103761 effective passes) train_loss=4.748 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=900 (0.109865 effective passes) train_loss=4.758 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=950 (0.115969 effective passes) train_loss=4.799 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.851 iter_dt=2.79e-01s lr=6.00e-02 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.762 val_pp=116.974 val_acc=0.250929 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.860 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.709 iter_dt=2.77e-01s lr=6.00e-02 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.698 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.659 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.679 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.745 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.736 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.738 iter_dt=2.78e-01s lr=6.00e-02 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.616 iter_dt=2.93e-01s lr=6.00e-02 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.733 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.621 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.924 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.464 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.849 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.500 iter_dt=2.81e-01s lr=6.00e-02 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.541 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.826 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.456 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.470 iter_dt=2.81e-01s lr=6.00e-02 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.599 iter_dt=2.79e-01s lr=6.00e-02 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.571 val_pp=96.684 val_acc=0.266104 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.601 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.601 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.575 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.538 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.580 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.313 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.486 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.528 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.493 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.764 iter_dt=2.78e-01s lr=6.00e-02 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.560 iter_dt=2.78e-01s lr=6.00e-02 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.319 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.730 iter_dt=2.78e-01s lr=6.00e-02 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.609 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.494 iter_dt=2.81e-01s lr=6.00e-02 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.375 iter_dt=2.81e-01s lr=6.00e-02 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.680 iter_dt=2.81e-01s lr=6.00e-02 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.600 iter_dt=2.82e-01s lr=6.00e-02 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.538 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.373 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.540 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3100 (0.378424 effective passes) train_loss=4.161 iter_dt=2.81e-01s lr=6.00e-02 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.434 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.516 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.582 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.490 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.673 iter_dt=2.81e-01s lr=6.00e-02 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.392 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.339 iter_dt=2.86e-01s lr=6.00e-02 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.443 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.542 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.599 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.563 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.402 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.419 iter_dt=2.77e-01s lr=6.00e-02 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.546 iter_dt=2.80e-01s lr=6.00e-02 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.348 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.428 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.492 iter_dt=2.78e-01s lr=6.00e-02 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.593 iter_dt=2.79e-01s lr=6.00e-02 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.447 iter_dt=2.79e-01s lr=6.00e-02 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.457 val_pp=86.244 val_acc=0.274332 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456/summary.json new file mode 100644 index 0000000..99c27a1 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [6.777387619018555, 5.844334602355957, 5.550934791564941, 5.476045608520508, 5.434511184692383, 5.092782497406006, 5.292604923248291, 4.968441009521484, 4.828104019165039, 4.875331878662109, 4.9657745361328125, 4.6918792724609375, 5.085966110229492, 4.971216678619385, 5.010965347290039, 4.838173866271973, 4.747869491577148, 4.757734298706055, 4.799452781677246, 4.851240634918213, 4.860150337219238, 4.709456443786621, 4.698463439941406, 4.659270763397217, 4.678989887237549, 4.744904518127441, 4.735501289367676, 4.738269805908203, 4.6157073974609375, 4.73347806930542, 4.620791435241699, 4.92420768737793, 4.4635114669799805, 4.849427223205566, 4.4996256828308105, 4.541390895843506, 4.825976848602295, 4.45644474029541, 4.470463752746582, 4.599381446838379, 4.601385593414307, 4.601446628570557, 4.574680805206299, 4.538341522216797, 4.580449104309082, 4.313079357147217, 4.485980987548828, 4.52764892578125, 4.493325233459473, 4.764386177062988, 4.559922695159912, 4.319311141967773, 4.7301025390625, 4.608814239501953, 4.493804931640625, 4.375072479248047, 4.679524898529053, 4.599964141845703, 4.537729263305664, 4.373368263244629, 4.540450572967529, 4.160916328430176, 4.433994293212891, 4.51555061340332, 4.582404136657715, 4.489991188049316, 4.672685623168945, 4.391645908355713, 4.338835716247559, 4.443374156951904, 4.541738510131836, 4.598578453063965, 4.563432216644287, 4.402200222015381, 4.418606758117676, 4.546417713165283, 4.347966194152832, 4.428362846374512, 4.492485046386719, 4.593419075012207, 4.447171211242676], "val_loss": [5.278108596801758, 4.948966979980469, 4.761952877044678, 4.571454048156738, 4.4571852684021], "val_pp": [195.99811590753652, 141.0287331811823, 116.97376441736354, 96.68429436191391, 86.24415368608321], "val_acc": [0.2044086456298828, 0.23703956604003906, 0.2509288787841797, 0.26610374450683594, 0.27433204650878906], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.278108596801758, "val_perplexity": 195.99811590753652, "val_accuracy": 0.2044086456298828, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 4.948966979980469, "val_perplexity": 141.0287331811823, "val_accuracy": 0.23703956604003906, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.761952877044678, "val_perplexity": 116.97376441736354, "val_accuracy": 0.2509288787841797, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.571454048156738, "val_perplexity": 96.68429436191391, "val_accuracy": 0.26610374450683594, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.4571852684021, "val_perplexity": 86.24415368608321, "val_accuracy": 0.27433204650878906, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.279149354, 0.282993746, 0.284928648, 0.282375614, 0.282256559, 0.282496265, 0.282355904, 0.285493869, 0.281306414, 0.287780353, 0.285936519, 0.279160167, 0.283525088, 0.282427285, 0.281966925, 0.28198058, 0.282057627, 0.279006429, 0.279035316, 0.27934979, 0.281713499, 0.277061196, 0.278922892, 0.278567004, 0.279513285, 0.27935932, 0.278842174, 0.27848984, 0.293078275, 0.279473098, 0.279754945, 0.282391439, 0.279336861, 0.279607345, 0.281273505, 0.278607808, 0.279248135, 0.279962846, 0.280662629, 0.27948883, 0.279541543, 0.279727122, 0.27934941, 0.27976197, 0.279056046, 0.278709968, 0.278819787, 0.2791789, 0.279580153, 0.278420864, 0.278459312, 0.278604308, 0.278171679, 0.278541815, 0.280711227, 0.281206009, 0.280521408, 0.282368613, 0.279924885, 0.279792724, 0.279605856, 0.28143843, 0.279534753, 0.279352139, 0.279470102, 0.28016908, 0.280717756, 0.280419565, 0.286497801, 0.278614413, 0.279695791, 0.279586661, 0.278753076, 0.280297883, 0.277469916, 0.28046585, 0.27922082, 0.278578526, 0.277933127, 0.279089749, 0.278827824], "train_time_seconds": 1148.5943301670006, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 233707.80174491255, "peak_memory_bytes": 6117270528, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.06, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "muon-pytorch", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.95, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": true, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.003, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456/run.log new file mode 100644 index 0000000..3412422 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456/run.log @@ -0,0 +1,140 @@ +Starting Experiment: sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 0.03, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Muon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + adamw_lr: 0.003 + adamw_lr_ratio: 0.1 + adamw_wd: 0.1 + lr: 0.03 + momentum: 0.95 + nesterov: True + ns_steps: 5 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.662 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=5.824 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=5.501 iter_dt=2.91e-01s lr=3.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=5.363 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=5.333 iter_dt=2.84e-01s lr=3.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.168 val_pp=175.532 val_acc=0.212906 +Train: Iter=300 (0.036622 effective passes) train_loss=5.054 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=350 (0.042725 effective passes) train_loss=5.160 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=400 (0.048829 effective passes) train_loss=4.787 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=450 (0.054932 effective passes) train_loss=4.599 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=500 (0.061036 effective passes) train_loss=4.627 iter_dt=2.86e-01s lr=3.00e-02 +>Eval: Iter=512 (0.062501 effective passes) val_loss=4.705 val_pp=110.539 val_acc=0.255857 +Train: Iter=550 (0.067140 effective passes) train_loss=4.684 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=600 (0.073243 effective passes) train_loss=4.426 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=650 (0.079347 effective passes) train_loss=4.802 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=700 (0.085451 effective passes) train_loss=4.681 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=750 (0.091554 effective passes) train_loss=4.691 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=800 (0.097658 effective passes) train_loss=4.515 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=850 (0.103761 effective passes) train_loss=4.455 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=900 (0.109865 effective passes) train_loss=4.394 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=950 (0.115969 effective passes) train_loss=4.481 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.525 iter_dt=2.86e-01s lr=3.00e-02 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.409 val_pp=82.167 val_acc=0.280989 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.533 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.338 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.364 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.300 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.308 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.353 iter_dt=3.01e-01s lr=3.00e-02 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.383 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.394 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.261 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.316 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.270 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.550 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.124 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.460 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.138 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.167 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.469 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.072 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.011 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.200 iter_dt=2.87e-01s lr=3.00e-02 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.191 val_pp=66.120 val_acc=0.300385 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.208 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.221 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.193 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.182 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.213 iter_dt=2.97e-01s lr=3.00e-02 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.917 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.121 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.110 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.115 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.366 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.156 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.949 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.366 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.224 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.137 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.997 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.290 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.198 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.154 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.003 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.165 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.809 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.033 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.111 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.170 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.047 iter_dt=2.92e-01s lr=3.00e-02 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.267 iter_dt=2.91e-01s lr=3.00e-02 +Train: Iter=3400 (0.415045 effective passes) train_loss=3.981 iter_dt=2.85e-01s lr=3.00e-02 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.951 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.039 iter_dt=2.92e-01s lr=3.00e-02 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.105 iter_dt=3.00e-01s lr=3.00e-02 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.208 iter_dt=2.94e-01s lr=3.00e-02 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.151 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.030 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.017 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.095 iter_dt=2.89e-01s lr=3.00e-02 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.938 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.044 iter_dt=2.90e-01s lr=3.00e-02 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.118 iter_dt=2.92e-01s lr=3.00e-02 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.133 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.044 iter_dt=2.90e-01s lr=3.00e-02 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.044 val_pp=57.034 val_acc=0.313597 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456/summary.json new file mode 100644 index 0000000..c2d9d85 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [6.6617608070373535, 5.824339866638184, 5.501289367675781, 5.362716197967529, 5.332653999328613, 5.054283142089844, 5.160220146179199, 4.786879539489746, 4.598504066467285, 4.626934051513672, 4.683712959289551, 4.425976753234863, 4.802138805389404, 4.681285858154297, 4.69057559967041, 4.515199184417725, 4.455206871032715, 4.394129276275635, 4.481179714202881, 4.524949073791504, 4.532565116882324, 4.338363170623779, 4.363523483276367, 4.300057411193848, 4.3082051277160645, 4.353421211242676, 4.383131980895996, 4.3942461013793945, 4.261031627655029, 4.316232681274414, 4.269717693328857, 4.5497918128967285, 4.123777389526367, 4.4601874351501465, 4.138162612915039, 4.166655540466309, 4.4691643714904785, 4.072012901306152, 4.010794639587402, 4.1999311447143555, 4.2078447341918945, 4.220701217651367, 4.193368911743164, 4.181970596313477, 4.2129106521606445, 3.91664981842041, 4.1209940910339355, 4.110225677490234, 4.114912033081055, 4.365564346313477, 4.155880928039551, 3.9492037296295166, 4.366342067718506, 4.224414825439453, 4.136787414550781, 3.996847629547119, 4.2902913093566895, 4.197877883911133, 4.153892517089844, 4.00257682800293, 4.165339469909668, 3.808955192565918, 4.032508850097656, 4.111040115356445, 4.1698102951049805, 4.047430992126465, 4.267433166503906, 3.9814343452453613, 3.9512529373168945, 4.038597106933594, 4.105208396911621, 4.207630157470703, 4.150579452514648, 4.030282974243164, 4.01740026473999, 4.095279693603516, 3.937725782394409, 4.04404354095459, 4.117799758911133, 4.132618427276611, 4.0437822341918945], "val_loss": [5.167823791503906, 4.705371856689453, 4.408752918243408, 4.191479682922363, 4.043658256530762], "val_pp": [175.53181619810817, 110.53903224899439, 82.16668703325017, 66.12036964881675, 57.03445372556012], "val_acc": [0.2129058837890625, 0.2558574676513672, 0.2809886932373047, 0.300384521484375, 0.3135967254638672], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.167823791503906, "val_perplexity": 175.53181619810817, "val_accuracy": 0.2129058837890625, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 4.705371856689453, "val_perplexity": 110.53903224899439, "val_accuracy": 0.2558574676513672, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.408752918243408, "val_perplexity": 82.16668703325017, "val_accuracy": 0.2809886932373047, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.191479682922363, "val_perplexity": 66.12036964881675, "val_accuracy": 0.300384521484375, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.043658256530762, "val_perplexity": 57.03445372556012, "val_accuracy": 0.3135967254638672, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.285153057, 0.285160652, 0.290501405, 0.28439492, 0.284407342, 0.284769149, 0.284499529, 0.284718545, 0.285568452, 0.285975394, 0.285235839, 0.285287545, 0.285823843, 0.285146818, 0.2851614, 0.285016987, 0.284902064, 0.285003639, 0.286251663, 0.285589586, 0.287850117, 0.289149551, 0.287487509, 0.285630947, 0.285814455, 0.30056262, 0.284968342, 0.290420724, 0.288060744, 0.285261163, 0.284886881, 0.287905896, 0.288858603, 0.285599786, 0.286490339, 0.286739314, 0.285796158, 0.287185181, 0.289000742, 0.28665776, 0.286168731, 0.285782179, 0.286754089, 0.285789809, 0.296690491, 0.286800018, 0.284657906, 0.286261503, 0.28870257, 0.288609105, 0.286529167, 0.286213606, 0.284913157, 0.285301522, 0.285383942, 0.284718611, 0.285048286, 0.286831644, 0.287661118, 0.285908542, 0.287698322, 0.286858785, 0.287008482, 0.283575876, 0.285290944, 0.292089327, 0.291306845, 0.284653663, 0.287630687, 0.291604029, 0.299975689, 0.294384847, 0.288112905, 0.288077141, 0.288668095, 0.288849674, 0.287576509, 0.290309315, 0.291646926, 0.287193677, 0.289602546], "train_time_seconds": 1175.974133417001, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 228266.4629875942, "peak_memory_bytes": 6117368832, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "muon", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.95, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": true, "muon_ns_steps": 5, "muon_lr_factor": 0.03, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456/run.log new file mode 100644 index 0000000..e3ecb8a --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456/run.log @@ -0,0 +1,247 @@ +Starting Experiment: sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.3, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'prodigy', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': True, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Using decoupled weight decay + +Optimizer: +Prodigy ( +Parameter Group 0 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 0.3 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.1 + +Parameter Group 1 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 0.3 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=10.238 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=6.14e-06 +Train: Iter=100 (0.012207 effective passes) train_loss=6.907 iter_dt=2.82e-01s lr=3.00e-01 +effective_lr=1.50e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.316 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=1.82e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.168 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=2.95e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=5.929 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=3.78e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.788 val_pp=326.202 val_acc=0.179335 +Train: Iter=300 (0.036622 effective passes) train_loss=5.558 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=4.43e-04 +Train: Iter=350 (0.042725 effective passes) train_loss=5.682 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=4.99e-04 +Train: Iter=400 (0.048829 effective passes) train_loss=5.337 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=5.47e-04 +Train: Iter=450 (0.054932 effective passes) train_loss=5.105 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=5.76e-04 +Train: Iter=500 (0.061036 effective passes) train_loss=5.051 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=6.07e-04 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.106 val_pp=164.941 val_acc=0.218599 +Train: Iter=550 (0.067140 effective passes) train_loss=5.167 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=6.37e-04 +Train: Iter=600 (0.073243 effective passes) train_loss=4.714 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=6.65e-04 +Train: Iter=650 (0.079347 effective passes) train_loss=5.089 iter_dt=2.82e-01s lr=3.00e-01 +effective_lr=6.84e-04 +Train: Iter=700 (0.085451 effective passes) train_loss=4.903 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=7.02e-04 +Train: Iter=750 (0.091554 effective passes) train_loss=4.889 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=7.19e-04 +Train: Iter=800 (0.097658 effective passes) train_loss=4.703 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=7.35e-04 +Train: Iter=850 (0.103761 effective passes) train_loss=4.628 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=7.49e-04 +Train: Iter=900 (0.109865 effective passes) train_loss=4.555 iter_dt=2.82e-01s lr=3.00e-01 +effective_lr=7.62e-04 +Train: Iter=950 (0.115969 effective passes) train_loss=4.634 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=7.75e-04 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.634 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=7.87e-04 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.549 val_pp=94.495 val_acc=0.270250 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.658 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=7.98e-04 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.465 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=8.08e-04 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.493 iter_dt=2.82e-01s lr=3.00e-01 +effective_lr=8.18e-04 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.409 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=8.27e-04 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.435 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=8.36e-04 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.468 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=8.44e-04 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.497 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=8.52e-04 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.495 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=8.59e-04 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.357 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=8.66e-04 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.427 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=8.72e-04 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.362 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=8.78e-04 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.621 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=8.84e-04 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.194 iter_dt=2.84e-01s lr=3.00e-01 +effective_lr=8.89e-04 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.556 iter_dt=2.85e-01s lr=3.00e-01 +effective_lr=8.95e-04 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.221 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=8.99e-04 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.261 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.04e-04 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.554 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.08e-04 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.145 iter_dt=2.78e-01s lr=3.00e-01 +effective_lr=9.12e-04 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.103 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.16e-04 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.295 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.20e-04 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.271 val_pp=71.619 val_acc=0.292839 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.288 iter_dt=2.78e-01s lr=3.00e-01 +effective_lr=9.24e-04 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.297 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.27e-04 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.258 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=9.30e-04 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.241 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.33e-04 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.300 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.36e-04 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.996 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.38e-04 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.180 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.41e-04 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.189 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.43e-04 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.184 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.46e-04 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.447 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.48e-04 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.223 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.50e-04 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.001 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.52e-04 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.434 iter_dt=2.82e-01s lr=3.00e-01 +effective_lr=9.54e-04 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.306 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.56e-04 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.199 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=9.57e-04 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.053 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.59e-04 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.374 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.60e-04 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.287 iter_dt=2.78e-01s lr=3.00e-01 +effective_lr=9.62e-04 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.222 iter_dt=2.78e-01s lr=3.00e-01 +effective_lr=9.63e-04 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.056 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.64e-04 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.242 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=9.66e-04 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.862 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.67e-04 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.103 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.68e-04 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.169 iter_dt=2.78e-01s lr=3.00e-01 +effective_lr=9.69e-04 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.232 iter_dt=2.78e-01s lr=3.00e-01 +effective_lr=9.70e-04 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.108 iter_dt=2.83e-01s lr=3.00e-01 +effective_lr=9.71e-04 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.330 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=9.72e-04 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.034 iter_dt=2.82e-01s lr=3.00e-01 +effective_lr=9.73e-04 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.003 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=9.74e-04 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.111 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.74e-04 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.170 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.75e-04 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.239 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.76e-04 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.199 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.76e-04 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.067 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=9.77e-04 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.052 iter_dt=2.85e-01s lr=3.00e-01 +effective_lr=9.78e-04 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.165 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.78e-04 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.988 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.79e-04 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.088 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.79e-04 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.148 iter_dt=2.79e-01s lr=3.00e-01 +effective_lr=9.80e-04 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.189 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=9.80e-04 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.080 iter_dt=2.78e-01s lr=3.00e-01 +effective_lr=9.81e-04 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.091 val_pp=59.827 val_acc=0.307278 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456/summary.json new file mode 100644 index 0000000..4b7eff3 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [10.23797607421875, 6.906857490539551, 6.315671920776367, 6.167749404907227, 5.9287919998168945, 5.557915687561035, 5.682114601135254, 5.33651065826416, 5.1046648025512695, 5.050976276397705, 5.167304992675781, 4.7142181396484375, 5.089312553405762, 4.903171539306641, 4.889158725738525, 4.703258514404297, 4.627804756164551, 4.555070400238037, 4.633928298950195, 4.633649826049805, 4.65848445892334, 4.465238571166992, 4.492995262145996, 4.409391403198242, 4.434569358825684, 4.467528343200684, 4.496933937072754, 4.4946699142456055, 4.356584548950195, 4.426579475402832, 4.361588478088379, 4.62059211730957, 4.193721294403076, 4.555793762207031, 4.2207136154174805, 4.2607526779174805, 4.554454803466797, 4.144697666168213, 4.103415489196777, 4.294730186462402, 4.287717819213867, 4.296683311462402, 4.257646560668945, 4.241310119628906, 4.299562454223633, 3.9962105751037598, 4.18002986907959, 4.18869686126709, 4.184278964996338, 4.447457313537598, 4.22345495223999, 4.00058126449585, 4.434243679046631, 4.306262493133545, 4.199440956115723, 4.05311918258667, 4.374418258666992, 4.286769866943359, 4.222163200378418, 4.056042194366455, 4.241790771484375, 3.8616366386413574, 4.102505683898926, 4.169375419616699, 4.231907844543457, 4.1084370613098145, 4.329565048217773, 4.0336995124816895, 4.002940654754639, 4.11135196685791, 4.169610977172852, 4.2389235496521, 4.1992082595825195, 4.0670623779296875, 4.052322864532471, 4.1653876304626465, 3.988095760345459, 4.0876569747924805, 4.1477789878845215, 4.189259052276611, 4.0797247886657715], "val_loss": [5.787520885467529, 5.105589866638184, 4.548550128936768, 4.271369934082031, 4.091464042663574], "val_pp": [326.2020558131375, 164.9407687779071, 94.49501382416213, 71.61947691543894, 59.82725285657247], "val_acc": [0.1793346405029297, 0.2185993194580078, 0.2702503204345703, 0.29283905029296875, 0.3072776794433594], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.787520885467529, "val_perplexity": 326.2020558131375, "val_accuracy": 0.1793346405029297, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 5.105589866638184, "val_perplexity": 164.9407687779071, "val_accuracy": 0.2185993194580078, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.548550128936768, "val_perplexity": 94.49501382416213, "val_accuracy": 0.2702503204345703, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.271369934082031, "val_perplexity": 71.61947691543894, "val_accuracy": 0.29283905029296875, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.091464042663574, "val_perplexity": 59.82725285657247, "val_accuracy": 0.3072776794433594, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.278673019, 0.282239367, 0.279148404, 0.279809593, 0.280331537, 0.279904645, 0.280033699, 0.279259263, 0.280575425, 0.281086582, 0.280195879, 0.280059921, 0.28176562, 0.280823987, 0.279867613, 0.279800714, 0.281032825, 0.281860716, 0.281081013, 0.279635528, 0.279524669, 0.280153314, 0.281687149, 0.279352403, 0.279482921, 0.278737504, 0.279227452, 0.279881484, 0.280139687, 0.278538969, 0.278821942, 0.278928298, 0.283523525, 0.285208554, 0.279360994, 0.278914575, 0.278928711, 0.277964865, 0.279456476, 0.278835957, 0.278464503, 0.280069088, 0.280912279, 0.279143231, 0.279361425, 0.279335693, 0.279487114, 0.279227681, 0.279474732, 0.279266131, 0.278995148, 0.280263382, 0.282201403, 0.279958303, 0.280927848, 0.280341931, 0.278810076, 0.278080314, 0.278139109, 0.279305489, 0.281385663, 0.279925015, 0.279875025, 0.278048599, 0.278323281, 0.282503148, 0.280785215, 0.281512479, 0.280936986, 0.278646603, 0.280102665, 0.280290996, 0.280291602, 0.280524974, 0.284861198, 0.280310947, 0.278618629, 0.279110351, 0.278729138, 0.279507521, 0.278303034], "train_time_seconds": 1147.7304891460017, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 233883.70226161394, "peak_memory_bytes": 6451916800, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.3, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "prodigy", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": true, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456/run.log new file mode 100644 index 0000000..d51d9ef --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456/run.log @@ -0,0 +1,146 @@ +Starting Experiment: sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion-light', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +ScionLight ( +Parameter Group 0 + lr: 0.0003 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.0003 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.337 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.690 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.537 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.580 iter_dt=2.94e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.598 iter_dt=2.91e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.452 val_pp=634.052 val_acc=0.142305 +Train: Iter=300 (0.036622 effective passes) train_loss=6.625 iter_dt=2.90e-01s lr=3.00e-04 +Train: Iter=350 (0.042725 effective passes) train_loss=6.661 iter_dt=2.99e-01s lr=3.00e-04 +Train: Iter=400 (0.048829 effective passes) train_loss=6.425 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=450 (0.054932 effective passes) train_loss=6.259 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=500 (0.061036 effective passes) train_loss=6.269 iter_dt=2.86e-01s lr=3.00e-04 +>Eval: Iter=512 (0.062501 effective passes) val_loss=6.336 val_pp=564.683 val_acc=0.145407 +Train: Iter=550 (0.067140 effective passes) train_loss=6.603 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=600 (0.073243 effective passes) train_loss=5.972 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=650 (0.079347 effective passes) train_loss=6.548 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=700 (0.085451 effective passes) train_loss=6.388 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=750 (0.091554 effective passes) train_loss=6.350 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=800 (0.097658 effective passes) train_loss=6.264 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=850 (0.103761 effective passes) train_loss=6.295 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=900 (0.109865 effective passes) train_loss=6.292 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=950 (0.115969 effective passes) train_loss=6.261 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1000 (0.122072 effective passes) train_loss=6.169 iter_dt=2.85e-01s lr=3.00e-04 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=6.414 val_pp=610.194 val_acc=0.142185 +Train: Iter=1050 (0.128176 effective passes) train_loss=6.393 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1100 (0.134279 effective passes) train_loss=6.306 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=1150 (0.140383 effective passes) train_loss=6.124 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=1200 (0.146487 effective passes) train_loss=6.160 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=1250 (0.152590 effective passes) train_loss=6.254 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1300 (0.158694 effective passes) train_loss=6.348 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1350 (0.164797 effective passes) train_loss=6.198 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1400 (0.170901 effective passes) train_loss=6.329 iter_dt=2.89e-01s lr=3.00e-04 +Train: Iter=1450 (0.177005 effective passes) train_loss=6.241 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1500 (0.183108 effective passes) train_loss=6.430 iter_dt=2.89e-01s lr=3.00e-04 +Train: Iter=1550 (0.189212 effective passes) train_loss=6.203 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1600 (0.195315 effective passes) train_loss=6.490 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1650 (0.201419 effective passes) train_loss=6.133 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1700 (0.207523 effective passes) train_loss=6.648 iter_dt=2.92e-01s lr=3.00e-04 +Train: Iter=1750 (0.213626 effective passes) train_loss=6.291 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=1800 (0.219730 effective passes) train_loss=6.180 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1850 (0.225834 effective passes) train_loss=6.390 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1900 (0.231937 effective passes) train_loss=6.212 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1950 (0.238041 effective passes) train_loss=6.375 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=2000 (0.244144 effective passes) train_loss=6.475 iter_dt=2.88e-01s lr=3.00e-04 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=6.294 val_pp=541.089 val_acc=0.146240 +Train: Iter=2050 (0.250248 effective passes) train_loss=6.552 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2100 (0.256352 effective passes) train_loss=6.330 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=2150 (0.262455 effective passes) train_loss=6.417 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2200 (0.268559 effective passes) train_loss=6.074 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=2250 (0.274662 effective passes) train_loss=6.178 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=2300 (0.280766 effective passes) train_loss=6.254 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=2350 (0.286870 effective passes) train_loss=6.077 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=2400 (0.292973 effective passes) train_loss=6.600 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=2450 (0.299077 effective passes) train_loss=6.104 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=2500 (0.305180 effective passes) train_loss=6.455 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=2550 (0.311284 effective passes) train_loss=6.559 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=2600 (0.317388 effective passes) train_loss=6.030 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=2650 (0.323491 effective passes) train_loss=6.160 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=2700 (0.329595 effective passes) train_loss=6.359 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=2750 (0.335698 effective passes) train_loss=6.320 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=2800 (0.341802 effective passes) train_loss=6.246 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=2850 (0.347906 effective passes) train_loss=6.442 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=2900 (0.354009 effective passes) train_loss=6.435 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=2950 (0.360113 effective passes) train_loss=6.248 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=3000 (0.366217 effective passes) train_loss=6.096 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=3050 (0.372320 effective passes) train_loss=6.414 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3100 (0.378424 effective passes) train_loss=5.788 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=3150 (0.384527 effective passes) train_loss=6.209 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3200 (0.390631 effective passes) train_loss=6.203 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3250 (0.396735 effective passes) train_loss=6.362 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=3300 (0.402838 effective passes) train_loss=6.462 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3350 (0.408942 effective passes) train_loss=6.288 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=3400 (0.415045 effective passes) train_loss=6.100 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=3450 (0.421149 effective passes) train_loss=6.030 iter_dt=2.82e-01s lr=3.00e-04 +Train: Iter=3500 (0.427253 effective passes) train_loss=6.156 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=3550 (0.433356 effective passes) train_loss=6.399 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=3600 (0.439460 effective passes) train_loss=6.482 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=3650 (0.445563 effective passes) train_loss=6.377 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=3700 (0.451667 effective passes) train_loss=6.222 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=3750 (0.457771 effective passes) train_loss=6.341 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=3800 (0.463874 effective passes) train_loss=6.652 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3850 (0.469978 effective passes) train_loss=6.156 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=3900 (0.476081 effective passes) train_loss=6.089 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=3950 (0.482185 effective passes) train_loss=6.243 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=4000 (0.488289 effective passes) train_loss=6.564 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=4050 (0.494392 effective passes) train_loss=6.104 iter_dt=2.85e-01s lr=3.00e-04 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=6.240 val_pp=512.879 val_acc=0.148157 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456/summary.json new file mode 100644 index 0000000..d1e1ac4 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.33674430847168, 6.689892292022705, 6.5373454093933105, 6.57993221282959, 6.598487854003906, 6.625079154968262, 6.661044597625732, 6.425187110900879, 6.258632659912109, 6.268501281738281, 6.602532386779785, 5.972364902496338, 6.547731399536133, 6.388250827789307, 6.350353240966797, 6.263527870178223, 6.295450687408447, 6.292353630065918, 6.261073112487793, 6.168988227844238, 6.393195152282715, 6.30640983581543, 6.124354362487793, 6.160013198852539, 6.253574848175049, 6.3475871086120605, 6.197858810424805, 6.32875919342041, 6.241320610046387, 6.430267333984375, 6.203042030334473, 6.48955774307251, 6.133274078369141, 6.6478424072265625, 6.290705680847168, 6.179594993591309, 6.389681816101074, 6.2124128341674805, 6.375380516052246, 6.474571228027344, 6.551631927490234, 6.330012321472168, 6.417219161987305, 6.074260711669922, 6.177668571472168, 6.254308700561523, 6.077495098114014, 6.599891185760498, 6.104021072387695, 6.455093860626221, 6.558863639831543, 6.02950382232666, 6.159542083740234, 6.358748435974121, 6.320445537567139, 6.24632453918457, 6.4420576095581055, 6.435103416442871, 6.2480058670043945, 6.096417427062988, 6.414034843444824, 5.787714958190918, 6.209200859069824, 6.202784538269043, 6.3617095947265625, 6.4615702629089355, 6.287718772888184, 6.100240707397461, 6.030329704284668, 6.1555986404418945, 6.399240493774414, 6.481882572174072, 6.377315521240234, 6.222288131713867, 6.341155052185059, 6.651858329772949, 6.156097412109375, 6.089262962341309, 6.2427520751953125, 6.563990592956543, 6.104160308837891], "val_loss": [6.452136039733887, 6.336269378662109, 6.41378116607666, 6.293587684631348, 6.240045070648193], "val_pp": [634.0524626588718, 564.6833415248443, 610.1939465694576, 541.088824551898, 512.8794735730139], "val_acc": [0.1423053741455078, 0.14540672302246094, 0.14218521118164062, 0.146240234375, 0.14815711975097656], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.452136039733887, "val_perplexity": 634.0524626588718, "val_accuracy": 0.1423053741455078, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 6.336269378662109, "val_perplexity": 564.6833415248443, "val_accuracy": 0.14540672302246094, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 6.41378116607666, "val_perplexity": 610.1939465694576, "val_accuracy": 0.14218521118164062, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 6.293587684631348, "val_perplexity": 541.088824551898, "val_accuracy": 0.146240234375, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 6.240045070648193, "val_perplexity": 512.8794735730139, "val_accuracy": 0.14815711975097656, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.285615946, 0.284990134, 0.28644794, 0.294306309, 0.290563315, 0.289635448, 0.298716155, 0.287192777, 0.286567736, 0.2861919, 0.286136832, 0.28648843, 0.286620198, 0.287134136, 0.286966004, 0.286429338, 0.286737536, 0.284694228, 0.285975691, 0.284916989, 0.287678333, 0.284791402, 0.285147517, 0.287104144, 0.286387455, 0.287577109, 0.28586696, 0.289269186, 0.288212718, 0.289422861, 0.288138324, 0.287689876, 0.287901357, 0.291737401, 0.285162613, 0.287536181, 0.287550766, 0.287821422, 0.287798797, 0.288297877, 0.286857724, 0.285465295, 0.285534788, 0.284822012, 0.284447821, 0.283431483, 0.284401129, 0.284135196, 0.283099703, 0.284421644, 0.284924774, 0.285254028, 0.28437258, 0.283306626, 0.28441139, 0.283835198, 0.283725559, 0.283756827, 0.28341678, 0.28325408, 0.284942489, 0.283824518, 0.284714256, 0.284652243, 0.283715895, 0.285632711, 0.282581308, 0.282971549, 0.282068272, 0.283182274, 0.282503585, 0.282867065, 0.283611693, 0.287991771, 0.283659764, 0.284697783, 0.285071335, 0.283998366, 0.283652932, 0.284933277, 0.284805445], "train_time_seconds": 1171.1609263130017, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 229204.5866361653, "peak_memory_bytes": 5909968384, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "scion-light", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456/run.log new file mode 100644 index 0000000..4568c8c --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456/run.log @@ -0,0 +1,146 @@ +Starting Experiment: sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +Scion ( +Parameter Group 0 + lr: 0.0003 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.0003 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.068 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.438 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.293 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.345 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.366 iter_dt=2.88e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.255 val_pp=520.603 val_acc=0.140072 +Train: Iter=300 (0.036622 effective passes) train_loss=6.215 iter_dt=2.91e-01s lr=3.00e-04 +Train: Iter=350 (0.042725 effective passes) train_loss=6.420 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=400 (0.048829 effective passes) train_loss=6.229 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=450 (0.054932 effective passes) train_loss=6.070 iter_dt=2.89e-01s lr=3.00e-04 +Train: Iter=500 (0.061036 effective passes) train_loss=6.009 iter_dt=2.87e-01s lr=3.00e-04 +>Eval: Iter=512 (0.062501 effective passes) val_loss=6.164 val_pp=475.152 val_acc=0.145535 +Train: Iter=550 (0.067140 effective passes) train_loss=6.303 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=600 (0.073243 effective passes) train_loss=5.813 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=650 (0.079347 effective passes) train_loss=6.311 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=700 (0.085451 effective passes) train_loss=6.090 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=750 (0.091554 effective passes) train_loss=6.104 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=800 (0.097658 effective passes) train_loss=6.024 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=850 (0.103761 effective passes) train_loss=6.034 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=900 (0.109865 effective passes) train_loss=5.985 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=950 (0.115969 effective passes) train_loss=5.963 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=1000 (0.122072 effective passes) train_loss=5.954 iter_dt=2.86e-01s lr=3.00e-04 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=6.059 val_pp=427.777 val_acc=0.148499 +Train: Iter=1050 (0.128176 effective passes) train_loss=6.140 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1100 (0.134279 effective passes) train_loss=6.035 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=1150 (0.140383 effective passes) train_loss=5.909 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=1200 (0.146487 effective passes) train_loss=5.938 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1250 (0.152590 effective passes) train_loss=6.019 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1300 (0.158694 effective passes) train_loss=6.086 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1350 (0.164797 effective passes) train_loss=5.989 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1400 (0.170901 effective passes) train_loss=6.012 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1450 (0.177005 effective passes) train_loss=5.943 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1500 (0.183108 effective passes) train_loss=6.140 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1550 (0.189212 effective passes) train_loss=5.924 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1600 (0.195315 effective passes) train_loss=6.259 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=1650 (0.201419 effective passes) train_loss=5.887 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1700 (0.207523 effective passes) train_loss=6.295 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1750 (0.213626 effective passes) train_loss=5.973 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=1800 (0.219730 effective passes) train_loss=5.930 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=1850 (0.225834 effective passes) train_loss=6.123 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=1900 (0.231937 effective passes) train_loss=5.895 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=1950 (0.238041 effective passes) train_loss=6.107 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2000 (0.244144 effective passes) train_loss=6.133 iter_dt=2.86e-01s lr=3.00e-04 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=6.075 val_pp=434.811 val_acc=0.149952 +Train: Iter=2050 (0.250248 effective passes) train_loss=6.210 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2100 (0.256352 effective passes) train_loss=6.103 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2150 (0.262455 effective passes) train_loss=6.146 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2200 (0.268559 effective passes) train_loss=5.926 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2250 (0.274662 effective passes) train_loss=5.991 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2300 (0.280766 effective passes) train_loss=6.081 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2350 (0.286870 effective passes) train_loss=5.899 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=2400 (0.292973 effective passes) train_loss=6.340 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2450 (0.299077 effective passes) train_loss=5.931 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2500 (0.305180 effective passes) train_loss=6.231 iter_dt=2.93e-01s lr=3.00e-04 +Train: Iter=2550 (0.311284 effective passes) train_loss=6.321 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2600 (0.317388 effective passes) train_loss=5.826 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2650 (0.323491 effective passes) train_loss=6.034 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2700 (0.329595 effective passes) train_loss=6.212 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2750 (0.335698 effective passes) train_loss=6.159 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=2800 (0.341802 effective passes) train_loss=6.052 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2850 (0.347906 effective passes) train_loss=6.250 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2900 (0.354009 effective passes) train_loss=6.236 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=2950 (0.360113 effective passes) train_loss=6.052 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3000 (0.366217 effective passes) train_loss=5.913 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=3050 (0.372320 effective passes) train_loss=6.193 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3100 (0.378424 effective passes) train_loss=5.563 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3150 (0.384527 effective passes) train_loss=5.994 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3200 (0.390631 effective passes) train_loss=5.992 iter_dt=2.91e-01s lr=3.00e-04 +Train: Iter=3250 (0.396735 effective passes) train_loss=6.134 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3300 (0.402838 effective passes) train_loss=6.180 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=3350 (0.408942 effective passes) train_loss=6.064 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=3400 (0.415045 effective passes) train_loss=5.894 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=3450 (0.421149 effective passes) train_loss=5.831 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3500 (0.427253 effective passes) train_loss=5.956 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=3550 (0.433356 effective passes) train_loss=6.192 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3600 (0.439460 effective passes) train_loss=6.262 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3650 (0.445563 effective passes) train_loss=6.175 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3700 (0.451667 effective passes) train_loss=6.016 iter_dt=2.86e-01s lr=3.00e-04 +Train: Iter=3750 (0.457771 effective passes) train_loss=6.121 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3800 (0.463874 effective passes) train_loss=6.448 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3850 (0.469978 effective passes) train_loss=5.962 iter_dt=2.87e-01s lr=3.00e-04 +Train: Iter=3900 (0.476081 effective passes) train_loss=5.912 iter_dt=2.90e-01s lr=3.00e-04 +Train: Iter=3950 (0.482185 effective passes) train_loss=6.080 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=4000 (0.488289 effective passes) train_loss=6.398 iter_dt=2.93e-01s lr=3.00e-04 +Train: Iter=4050 (0.494392 effective passes) train_loss=5.952 iter_dt=2.88e-01s lr=3.00e-04 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=6.072 val_pp=433.616 val_acc=0.151756 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456/summary.json new file mode 100644 index 0000000..defbb55 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.068050861358643, 6.438355445861816, 6.29287576675415, 6.3450703620910645, 6.365862846374512, 6.215234279632568, 6.419804573059082, 6.228850364685059, 6.0700507164001465, 6.009476661682129, 6.30311918258667, 5.812990188598633, 6.3107147216796875, 6.09019660949707, 6.103971004486084, 6.024131774902344, 6.034391403198242, 5.984778881072998, 5.962920188903809, 5.953911781311035, 6.13960075378418, 6.034769058227539, 5.909461975097656, 5.93829345703125, 6.019379615783691, 6.085962295532227, 5.9886932373046875, 6.012336730957031, 5.9432783126831055, 6.140162467956543, 5.924415588378906, 6.258528232574463, 5.887404918670654, 6.295190334320068, 5.973418235778809, 5.929715156555176, 6.123244285583496, 5.894794940948486, 6.107370376586914, 6.132659435272217, 6.210149765014648, 6.102579116821289, 6.1460466384887695, 5.926255226135254, 5.991426944732666, 6.081372261047363, 5.899357795715332, 6.339741230010986, 5.931046485900879, 6.231234550476074, 6.321049690246582, 5.826025485992432, 6.0339202880859375, 6.211821556091309, 6.158863067626953, 6.052461624145508, 6.2503662109375, 6.236417770385742, 6.052042484283447, 5.912615776062012, 6.193409442901611, 5.563289642333984, 5.994320392608643, 5.992412567138672, 6.1340742111206055, 6.179794788360596, 6.064431190490723, 5.894491195678711, 5.831384658813477, 5.955809116363525, 6.192151069641113, 6.261920928955078, 6.174585819244385, 6.016330718994141, 6.120514869689941, 6.447704315185547, 5.961887836456299, 5.911600112915039, 6.080057621002197, 6.397638320922852, 5.952073097229004], "val_loss": [6.25499153137207, 6.163638591766357, 6.05860710144043, 6.074914932250977, 6.072164535522461], "val_pp": [520.6027755209898, 475.1518540963069, 427.77742535588936, 434.810735647104, 433.6164775202488], "val_acc": [0.14007186889648438, 0.14553451538085938, 0.14849853515625, 0.14995193481445312, 0.15175628662109375], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.25499153137207, "val_perplexity": 520.6027755209898, "val_accuracy": 0.14007186889648438, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 6.163638591766357, "val_perplexity": 475.1518540963069, "val_accuracy": 0.14553451538085938, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 6.05860710144043, "val_perplexity": 427.77742535588936, "val_accuracy": 0.14849853515625, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 6.074914932250977, "val_perplexity": 434.810735647104, "val_accuracy": 0.14995193481445312, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 6.072164535522461, "val_perplexity": 433.6164775202488, "val_accuracy": 0.15175628662109375, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.283662477, 0.285696984, 0.286175455, 0.287499233, 0.288253834, 0.29093522, 0.287142033, 0.287065084, 0.288529304, 0.287336389, 0.286401831, 0.288312844, 0.28575424, 0.28605144, 0.285427371, 0.285786585, 0.285573344, 0.286142876, 0.287093005, 0.285721035, 0.286371907, 0.286789761, 0.28681028, 0.286449725, 0.288053687, 0.28647766, 0.286232201, 0.285947588, 0.286410659, 0.286338704, 0.287775591, 0.287502177, 0.285588638, 0.286099881, 0.285894729, 0.285275832, 0.285254867, 0.287267713, 0.285990376, 0.285629688, 0.285654374, 0.287026927, 0.286025228, 0.286782904, 0.285979743, 0.285862125, 0.285474068, 0.285782354, 0.285680308, 0.293349599, 0.286620771, 0.285629366, 0.285932553, 0.286905537, 0.286281339, 0.28669341, 0.286823219, 0.287421271, 0.286073708, 0.288350377, 0.286635929, 0.286663839, 0.286706591, 0.290862973, 0.286690588, 0.28818537, 0.287556657, 0.287932693, 0.286657649, 0.28785065, 0.287464839, 0.286196966, 0.28586925, 0.2861357, 0.286653421, 0.286743994, 0.28735014, 0.289884096, 0.28816978, 0.293499073, 0.287743419], "train_time_seconds": 1175.894496335001, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 228281.92226143845, "peak_memory_bytes": 6039584256, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "scion", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456/run.log new file mode 100644 index 0000000..ecc88e9 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456/run.log @@ -0,0 +1,159 @@ +Starting Experiment: sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.9999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdamWScheduleFree ( +Parameter Group 0 + betas: (0.9, 0.9999) + eps: 1e-08 + foreach: True + k: 0 + lr: 0.001 + lr_max: -1.0 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + betas: (0.9, 0.9999) + eps: 1e-08 + foreach: True + k: 0 + lr: 0.001 + lr_max: -1.0 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.243 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.466 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.221 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.203 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.082 iter_dt=2.72e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.023 val_pp=412.680 val_acc=0.167620 +Train: Iter=300 (0.036622 effective passes) train_loss=5.870 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.973 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=5.712 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=5.518 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=5.452 iter_dt=2.73e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.568 val_pp=261.798 val_acc=0.191311 +Train: Iter=550 (0.067140 effective passes) train_loss=5.664 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=5.115 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=5.622 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=5.377 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=5.374 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=5.161 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=5.175 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=5.111 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=5.121 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=5.078 iter_dt=2.74e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=5.125 val_pp=168.201 val_acc=0.217758 +Train: Iter=1050 (0.128176 effective passes) train_loss=5.206 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=5.012 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.948 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.926 iter_dt=2.78e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.909 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=5.004 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.944 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.899 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.820 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.885 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.778 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=5.100 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.659 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.984 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.678 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.666 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.942 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.594 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.561 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.734 iter_dt=2.72e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.691 val_pp=108.955 val_acc=0.251253 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.705 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.686 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.641 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.546 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.641 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.401 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.491 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.581 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.497 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.752 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.573 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.283 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.679 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.593 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.520 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.357 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.640 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.575 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.453 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.349 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.509 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=4.095 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.360 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.415 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.480 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.385 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.569 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.244 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.227 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.312 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.408 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.471 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.419 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.257 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.286 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.403 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.201 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.292 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.358 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.403 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.268 iter_dt=2.72e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.305 val_pp=74.039 val_acc=0.290781 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456/summary.json new file mode 100644 index 0000000..c19475d --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.24257755279541, 6.466488838195801, 6.221114158630371, 6.202880859375, 6.08197021484375, 5.869750499725342, 5.972587585449219, 5.711933135986328, 5.518000602722168, 5.452193260192871, 5.664089202880859, 5.114889144897461, 5.621656894683838, 5.376613616943359, 5.374216079711914, 5.160927772521973, 5.1754255294799805, 5.111083030700684, 5.120929718017578, 5.077611923217773, 5.206099033355713, 5.011783599853516, 4.948143005371094, 4.926140785217285, 4.909237861633301, 5.004223823547363, 4.944339752197266, 4.898647308349609, 4.819806098937988, 4.884904861450195, 4.777652740478516, 5.100492000579834, 4.659289360046387, 4.984468460083008, 4.677832126617432, 4.666483402252197, 4.941586494445801, 4.593779563903809, 4.561188697814941, 4.733819484710693, 4.704904079437256, 4.685788154602051, 4.640688419342041, 4.5458502769470215, 4.6412672996521, 4.400694847106934, 4.49051570892334, 4.580864906311035, 4.49687385559082, 4.752397060394287, 4.573113441467285, 4.2833099365234375, 4.679055213928223, 4.5930986404418945, 4.519967079162598, 4.356977462768555, 4.639819145202637, 4.575054168701172, 4.453146934509277, 4.349037170410156, 4.509214401245117, 4.094670295715332, 4.360269546508789, 4.414556503295898, 4.4799580574035645, 4.385265350341797, 4.568979263305664, 4.2437286376953125, 4.226591110229492, 4.31187105178833, 4.408134937286377, 4.471045017242432, 4.418515205383301, 4.25741720199585, 4.285869598388672, 4.402782440185547, 4.201391696929932, 4.291635513305664, 4.358216285705566, 4.402834892272949, 4.267521381378174], "val_loss": [6.022677421569824, 5.567577362060547, 5.125164031982422, 4.690937042236328, 4.304592132568359], "val_pp": [412.680369830492, 261.79810550128695, 168.20114998532256, 108.95488384486792, 74.0387968129434], "val_acc": [0.1676197052001953, 0.19131088256835938, 0.2177581787109375, 0.2512531280517578, 0.29078102111816406], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.022677421569824, "val_perplexity": 412.680369830492, "val_accuracy": 0.1676197052001953, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 5.567577362060547, "val_perplexity": 261.79810550128695, "val_accuracy": 0.19131088256835938, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 5.125164031982422, "val_perplexity": 168.20114998532256, "val_accuracy": 0.2177581787109375, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.690937042236328, "val_perplexity": 108.95488384486792, "val_accuracy": 0.2512531280517578, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.304592132568359, "val_perplexity": 74.0387968129434, "val_accuracy": 0.29078102111816406, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.272922577, 0.273507712, 0.272765476, 0.271754536, 0.27242545, 0.274647953, 0.272573778, 0.272989204, 0.273371742, 0.272760389, 0.274036618, 0.273212411, 0.273861204, 0.273703537, 0.274402278, 0.273940829, 0.273991062, 0.274117426, 0.273709024, 0.27387522, 0.273265965, 0.272423863, 0.273005985, 0.278387211, 0.272876918, 0.272567231, 0.274144621, 0.272564577, 0.274157289, 0.275499204, 0.275232493, 0.273586649, 0.272721208, 0.273084958, 0.272711518, 0.273752102, 0.273043687, 0.273196218, 0.278715183, 0.271632168, 0.27465256, 0.272393758, 0.272186372, 0.272686202, 0.272463061, 0.271953948, 0.271932341, 0.272534423, 0.271680838, 0.271676746, 0.27395316, 0.271944126, 0.274567579, 0.273039834, 0.271791797, 0.2727735, 0.271509068, 0.273126567, 0.272199087, 0.271179167, 0.280781872, 0.273481976, 0.271963516, 0.270953801, 0.272106324, 0.272489631, 0.271926011, 0.27187151, 0.267300534, 0.272091629, 0.27126842, 0.271087704, 0.271527192, 0.275015686, 0.272305409, 0.271169101, 0.271978544, 0.271756748, 0.273059631, 0.271999985, 0.27175557], "train_time_seconds": 1116.8707595270018, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 240346.03261856656, "peak_memory_bytes": 6175400960, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "none", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sf-adamw", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.9999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456/run.log new file mode 100644 index 0000000..a5b936e --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456/run.log @@ -0,0 +1,157 @@ +Starting Experiment: sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 7.29, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGDScheduleFree ( +Parameter Group 0 + foreach: True + k: 0 + lr: 7.29 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + foreach: True + k: 0 + lr: 7.29 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.460 iter_dt=2.73e-01s lr=7.29e+00 +Train: Iter=100 (0.012207 effective passes) train_loss=7.914 iter_dt=2.74e-01s lr=7.29e+00 +Train: Iter=150 (0.018311 effective passes) train_loss=7.844 iter_dt=2.72e-01s lr=7.29e+00 +Train: Iter=200 (0.024414 effective passes) train_loss=7.829 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=250 (0.030518 effective passes) train_loss=8.025 iter_dt=2.75e-01s lr=7.29e+00 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.614 val_pp=2027.040 val_acc=0.053434 +Train: Iter=300 (0.036622 effective passes) train_loss=7.872 iter_dt=2.73e-01s lr=7.29e+00 +Train: Iter=350 (0.042725 effective passes) train_loss=7.702 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=400 (0.048829 effective passes) train_loss=7.595 iter_dt=2.72e-01s lr=7.29e+00 +Train: Iter=450 (0.054932 effective passes) train_loss=7.604 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=500 (0.061036 effective passes) train_loss=8.115 iter_dt=2.70e-01s lr=7.29e+00 +>Eval: Iter=512 (0.062501 effective passes) val_loss=7.474 val_pp=1760.878 val_acc=0.059734 +Train: Iter=550 (0.067140 effective passes) train_loss=7.745 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=600 (0.073243 effective passes) train_loss=7.165 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=650 (0.079347 effective passes) train_loss=7.732 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=700 (0.085451 effective passes) train_loss=7.511 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=750 (0.091554 effective passes) train_loss=7.480 iter_dt=2.72e-01s lr=7.29e+00 +Train: Iter=800 (0.097658 effective passes) train_loss=8.234 iter_dt=2.79e-01s lr=7.29e+00 +Train: Iter=850 (0.103761 effective passes) train_loss=7.467 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=900 (0.109865 effective passes) train_loss=7.445 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=950 (0.115969 effective passes) train_loss=8.328 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=1000 (0.122072 effective passes) train_loss=7.271 iter_dt=2.71e-01s lr=7.29e+00 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=7.332 val_pp=1529.089 val_acc=0.084913 +Train: Iter=1050 (0.128176 effective passes) train_loss=7.482 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=1100 (0.134279 effective passes) train_loss=7.431 iter_dt=2.76e-01s lr=7.29e+00 +Train: Iter=1150 (0.140383 effective passes) train_loss=7.354 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=1200 (0.146487 effective passes) train_loss=7.219 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=1250 (0.152590 effective passes) train_loss=7.848 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=1300 (0.158694 effective passes) train_loss=7.812 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=1350 (0.164797 effective passes) train_loss=7.574 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=1400 (0.170901 effective passes) train_loss=7.360 iter_dt=2.72e-01s lr=7.29e+00 +Train: Iter=1450 (0.177005 effective passes) train_loss=7.215 iter_dt=2.69e-01s lr=7.29e+00 +Train: Iter=1500 (0.183108 effective passes) train_loss=8.048 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=1550 (0.189212 effective passes) train_loss=10.113 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=1600 (0.195315 effective passes) train_loss=7.383 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=1650 (0.201419 effective passes) train_loss=7.066 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=1700 (0.207523 effective passes) train_loss=7.618 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=1750 (0.213626 effective passes) train_loss=7.235 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=1800 (0.219730 effective passes) train_loss=7.287 iter_dt=2.73e-01s lr=7.29e+00 +Train: Iter=1850 (0.225834 effective passes) train_loss=7.231 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=1900 (0.231937 effective passes) train_loss=7.089 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=1950 (0.238041 effective passes) train_loss=7.491 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=2000 (0.244144 effective passes) train_loss=7.319 iter_dt=2.69e-01s lr=7.29e+00 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=7.188 val_pp=1323.053 val_acc=0.089077 +Train: Iter=2050 (0.250248 effective passes) train_loss=7.757 iter_dt=2.69e-01s lr=7.29e+00 +Train: Iter=2100 (0.256352 effective passes) train_loss=7.333 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=2150 (0.262455 effective passes) train_loss=7.246 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=2200 (0.268559 effective passes) train_loss=7.126 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=2250 (0.274662 effective passes) train_loss=7.118 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=2300 (0.280766 effective passes) train_loss=7.445 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=2350 (0.286870 effective passes) train_loss=7.118 iter_dt=2.69e-01s lr=7.29e+00 +Train: Iter=2400 (0.292973 effective passes) train_loss=7.360 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=2450 (0.299077 effective passes) train_loss=7.061 iter_dt=2.69e-01s lr=7.29e+00 +Train: Iter=2500 (0.305180 effective passes) train_loss=7.238 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=2550 (0.311284 effective passes) train_loss=7.577 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=2600 (0.317388 effective passes) train_loss=7.003 iter_dt=2.69e-01s lr=7.29e+00 +Train: Iter=2650 (0.323491 effective passes) train_loss=7.327 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=2700 (0.329595 effective passes) train_loss=7.497 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=2750 (0.335698 effective passes) train_loss=7.135 iter_dt=2.69e-01s lr=7.29e+00 +Train: Iter=2800 (0.341802 effective passes) train_loss=7.167 iter_dt=2.69e-01s lr=7.29e+00 +Train: Iter=2850 (0.347906 effective passes) train_loss=7.513 iter_dt=2.69e-01s lr=7.29e+00 +Train: Iter=2900 (0.354009 effective passes) train_loss=7.237 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=2950 (0.360113 effective passes) train_loss=7.276 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3000 (0.366217 effective passes) train_loss=7.055 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=3050 (0.372320 effective passes) train_loss=7.489 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3100 (0.378424 effective passes) train_loss=7.820 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=3150 (0.384527 effective passes) train_loss=7.126 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3200 (0.390631 effective passes) train_loss=7.126 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3250 (0.396735 effective passes) train_loss=7.130 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3300 (0.402838 effective passes) train_loss=7.407 iter_dt=2.87e-01s lr=7.29e+00 +Train: Iter=3350 (0.408942 effective passes) train_loss=7.120 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3400 (0.415045 effective passes) train_loss=7.722 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3450 (0.421149 effective passes) train_loss=8.052 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3500 (0.427253 effective passes) train_loss=7.158 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3550 (0.433356 effective passes) train_loss=7.856 iter_dt=2.74e-01s lr=7.29e+00 +Train: Iter=3600 (0.439460 effective passes) train_loss=7.351 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=3650 (0.445563 effective passes) train_loss=7.519 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=3700 (0.451667 effective passes) train_loss=7.244 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=3750 (0.457771 effective passes) train_loss=7.126 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=3800 (0.463874 effective passes) train_loss=7.649 iter_dt=2.70e-01s lr=7.29e+00 +Train: Iter=3850 (0.469978 effective passes) train_loss=7.070 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=3900 (0.476081 effective passes) train_loss=7.072 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=3950 (0.482185 effective passes) train_loss=7.074 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=4000 (0.488289 effective passes) train_loss=7.296 iter_dt=2.69e-01s lr=7.29e+00 +Train: Iter=4050 (0.494392 effective passes) train_loss=6.844 iter_dt=2.69e-01s lr=7.29e+00 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=6.992 val_pp=1088.043 val_acc=0.106506 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456/summary.json new file mode 100644 index 0000000..65b976a --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.459695816040039, 7.913730621337891, 7.843610763549805, 7.829135894775391, 8.025308609008789, 7.871789932250977, 7.7024126052856445, 7.59528923034668, 7.603880882263184, 8.114728927612305, 7.744956970214844, 7.165470123291016, 7.732234001159668, 7.511425018310547, 7.48015832901001, 8.234468460083008, 7.467293739318848, 7.445259094238281, 8.328145027160645, 7.271249294281006, 7.481549263000488, 7.431097984313965, 7.3543806076049805, 7.219298839569092, 7.848136901855469, 7.8119707107543945, 7.574356555938721, 7.360489845275879, 7.215150356292725, 8.048473358154297, 10.112939834594727, 7.3827643394470215, 7.0661163330078125, 7.618117332458496, 7.234678268432617, 7.287235260009766, 7.230802536010742, 7.088512897491455, 7.490799903869629, 7.319170951843262, 7.757392406463623, 7.333443641662598, 7.245689392089844, 7.12587833404541, 7.117501258850098, 7.444885730743408, 7.117912292480469, 7.360101699829102, 7.060979843139648, 7.238387584686279, 7.577495098114014, 7.003148078918457, 7.327083587646484, 7.49715518951416, 7.134832382202148, 7.166714668273926, 7.513265132904053, 7.237366676330566, 7.275899887084961, 7.055387496948242, 7.489350318908691, 7.819792747497559, 7.126196384429932, 7.126371383666992, 7.129946708679199, 7.406883239746094, 7.119815826416016, 7.721628189086914, 8.051515579223633, 7.1579389572143555, 7.855924606323242, 7.351378440856934, 7.518734931945801, 7.243540287017822, 7.1255269050598145, 7.649333477020264, 7.069672584533691, 7.072142601013184, 7.073507308959961, 7.295621395111084, 6.844316482543945], "val_loss": [7.614336967468262, 7.473572731018066, 7.332432270050049, 7.18770170211792, 6.992140293121338], "val_pp": [2027.0399306956288, 1760.877782745687, 1529.0888863205523, 1323.052528344451, 1088.042609550867], "val_acc": [0.05343437194824219, 0.059734344482421875, 0.08491325378417969, 0.08907699584960938, 0.10650634765625], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 7.614336967468262, "val_perplexity": 2027.0399306956288, "val_accuracy": 0.05343437194824219, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 7.473572731018066, "val_perplexity": 1760.877782745687, "val_accuracy": 0.059734344482421875, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 7.332432270050049, "val_perplexity": 1529.0888863205523, "val_accuracy": 0.08491325378417969, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 7.18770170211792, "val_perplexity": 1323.052528344451, "val_accuracy": 0.08907699584960938, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 6.992140293121338, "val_perplexity": 1088.042609550867, "val_accuracy": 0.10650634765625, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.272789681, 0.273943374, 0.271760391, 0.27094846, 0.274901786, 0.273134181, 0.271427019, 0.271926137, 0.269778343, 0.270320536, 0.270040703, 0.270853899, 0.270638643, 0.27148334, 0.271865853, 0.278914087, 0.27084347, 0.271360552, 0.27080103, 0.270563313, 0.270439206, 0.275838292, 0.270390758, 0.269723635, 0.271035814, 0.270212297, 0.269607496, 0.272086663, 0.269189849, 0.27067806, 0.271440668, 0.269698827, 0.270005965, 0.269871638, 0.270776331, 0.272503776, 0.271311536, 0.270612891, 0.26962326, 0.269414101, 0.269288638, 0.270740446, 0.270438713, 0.270913106, 0.270798399, 0.270361218, 0.268847429, 0.270631899, 0.268792369, 0.269692953, 0.270233292, 0.269230929, 0.270317464, 0.271085256, 0.269187603, 0.269080542, 0.269478203, 0.26994018, 0.269851467, 0.270526022, 0.269628125, 0.270531504, 0.270009518, 0.270274376, 0.270078354, 0.287271813, 0.270486736, 0.269657673, 0.269803517, 0.270037713, 0.274479764, 0.270811439, 0.270675419, 0.27076329, 0.271081965, 0.270435486, 0.270911812, 0.270710352, 0.271122296, 0.26941551, 0.269109834], "train_time_seconds": 1108.248907474, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 242215.85438945956, "peak_memory_bytes": 6039977472, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "none", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 7.29, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sf-sgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456/run.log new file mode 100644 index 0000000..7af8325 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456/run.log @@ -0,0 +1,151 @@ +Starting Experiment: sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.09, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGD ( +Parameter Group 0 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.09 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.09 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.292 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=7.689 iter_dt=2.69e-01s lr=9.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=7.567 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=7.721 iter_dt=2.69e-01s lr=9.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=7.588 iter_dt=2.69e-01s lr=9.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.568 val_pp=1935.208 val_acc=0.051807 +Train: Iter=300 (0.036622 effective passes) train_loss=7.692 iter_dt=2.70e-01s lr=9.00e-02 +Train: Iter=350 (0.042725 effective passes) train_loss=7.563 iter_dt=2.70e-01s lr=9.00e-02 +Train: Iter=400 (0.048829 effective passes) train_loss=7.522 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=450 (0.054932 effective passes) train_loss=7.464 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=500 (0.061036 effective passes) train_loss=7.381 iter_dt=2.68e-01s lr=9.00e-02 +>Eval: Iter=512 (0.062501 effective passes) val_loss=7.446 val_pp=1713.474 val_acc=0.052227 +Train: Iter=550 (0.067140 effective passes) train_loss=7.624 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=600 (0.073243 effective passes) train_loss=7.130 iter_dt=2.74e-01s lr=9.00e-02 +Train: Iter=650 (0.079347 effective passes) train_loss=7.596 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=700 (0.085451 effective passes) train_loss=7.345 iter_dt=2.74e-01s lr=9.00e-02 +Train: Iter=750 (0.091554 effective passes) train_loss=7.326 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=800 (0.097658 effective passes) train_loss=7.391 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=850 (0.103761 effective passes) train_loss=7.338 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=900 (0.109865 effective passes) train_loss=7.294 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=950 (0.115969 effective passes) train_loss=7.228 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=1000 (0.122072 effective passes) train_loss=7.240 iter_dt=2.68e-01s lr=9.00e-02 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=7.362 val_pp=1574.316 val_acc=0.049957 +Train: Iter=1050 (0.128176 effective passes) train_loss=7.425 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=1100 (0.134279 effective passes) train_loss=7.381 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=1150 (0.140383 effective passes) train_loss=7.204 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=1200 (0.146487 effective passes) train_loss=7.232 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=1250 (0.152590 effective passes) train_loss=7.276 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=1300 (0.158694 effective passes) train_loss=7.364 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=1350 (0.164797 effective passes) train_loss=7.217 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=1400 (0.170901 effective passes) train_loss=7.280 iter_dt=2.69e-01s lr=9.00e-02 +Train: Iter=1450 (0.177005 effective passes) train_loss=7.224 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=1500 (0.183108 effective passes) train_loss=7.324 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=1550 (0.189212 effective passes) train_loss=7.164 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=1600 (0.195315 effective passes) train_loss=7.403 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=1650 (0.201419 effective passes) train_loss=7.120 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=1700 (0.207523 effective passes) train_loss=7.480 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=1750 (0.213626 effective passes) train_loss=7.257 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=1800 (0.219730 effective passes) train_loss=7.180 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=1850 (0.225834 effective passes) train_loss=7.318 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=1900 (0.231937 effective passes) train_loss=7.179 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=1950 (0.238041 effective passes) train_loss=7.385 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=2000 (0.244144 effective passes) train_loss=7.359 iter_dt=2.66e-01s lr=9.00e-02 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=7.295 val_pp=1473.470 val_acc=0.057417 +Train: Iter=2050 (0.250248 effective passes) train_loss=7.426 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=2100 (0.256352 effective passes) train_loss=7.263 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=2150 (0.262455 effective passes) train_loss=7.308 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=2200 (0.268559 effective passes) train_loss=7.084 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=2250 (0.274662 effective passes) train_loss=7.161 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=2300 (0.280766 effective passes) train_loss=7.262 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=2350 (0.286870 effective passes) train_loss=7.032 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=2400 (0.292973 effective passes) train_loss=7.418 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=2450 (0.299077 effective passes) train_loss=6.993 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=2500 (0.305180 effective passes) train_loss=7.202 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=2550 (0.311284 effective passes) train_loss=7.406 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=2600 (0.317388 effective passes) train_loss=6.963 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=2650 (0.323491 effective passes) train_loss=7.071 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=2700 (0.329595 effective passes) train_loss=7.203 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=2750 (0.335698 effective passes) train_loss=7.121 iter_dt=2.61e-01s lr=9.00e-02 +Train: Iter=2800 (0.341802 effective passes) train_loss=7.113 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=2850 (0.347906 effective passes) train_loss=7.234 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=2900 (0.354009 effective passes) train_loss=7.172 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=2950 (0.360113 effective passes) train_loss=7.074 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=3000 (0.366217 effective passes) train_loss=6.886 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3050 (0.372320 effective passes) train_loss=7.227 iter_dt=2.69e-01s lr=9.00e-02 +Train: Iter=3100 (0.378424 effective passes) train_loss=6.599 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=3150 (0.384527 effective passes) train_loss=6.945 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=3200 (0.390631 effective passes) train_loss=6.938 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=3250 (0.396735 effective passes) train_loss=7.133 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=3300 (0.402838 effective passes) train_loss=7.157 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3350 (0.408942 effective passes) train_loss=7.027 iter_dt=2.69e-01s lr=9.00e-02 +Train: Iter=3400 (0.415045 effective passes) train_loss=6.879 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=3450 (0.421149 effective passes) train_loss=6.841 iter_dt=2.68e-01s lr=9.00e-02 +Train: Iter=3500 (0.427253 effective passes) train_loss=6.870 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3550 (0.433356 effective passes) train_loss=7.156 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3600 (0.439460 effective passes) train_loss=7.161 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3650 (0.445563 effective passes) train_loss=7.119 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3700 (0.451667 effective passes) train_loss=6.961 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3750 (0.457771 effective passes) train_loss=7.021 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3800 (0.463874 effective passes) train_loss=7.404 iter_dt=2.66e-01s lr=9.00e-02 +Train: Iter=3850 (0.469978 effective passes) train_loss=6.866 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3900 (0.476081 effective passes) train_loss=6.835 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=3950 (0.482185 effective passes) train_loss=6.974 iter_dt=2.65e-01s lr=9.00e-02 +Train: Iter=4000 (0.488289 effective passes) train_loss=7.325 iter_dt=2.67e-01s lr=9.00e-02 +Train: Iter=4050 (0.494392 effective passes) train_loss=6.804 iter_dt=2.66e-01s lr=9.00e-02 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=6.973 val_pp=1067.516 val_acc=0.103691 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456/summary.json new file mode 100644 index 0000000..e6f7889 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.292377471923828, 7.689275741577148, 7.566953659057617, 7.721213340759277, 7.587551116943359, 7.691600799560547, 7.563418388366699, 7.522185325622559, 7.46424674987793, 7.380739212036133, 7.624111175537109, 7.13021183013916, 7.595746994018555, 7.345041275024414, 7.325963973999023, 7.390666961669922, 7.337549209594727, 7.294057369232178, 7.228310585021973, 7.239927291870117, 7.424619674682617, 7.38132381439209, 7.203652381896973, 7.232407569885254, 7.275962829589844, 7.364242076873779, 7.217479705810547, 7.280245780944824, 7.224396705627441, 7.32373046875, 7.163725852966309, 7.402854919433594, 7.119736671447754, 7.47999382019043, 7.257137298583984, 7.180468559265137, 7.317662239074707, 7.178778648376465, 7.384860992431641, 7.3591108322143555, 7.425958633422852, 7.26265811920166, 7.308459281921387, 7.083766937255859, 7.1609296798706055, 7.261899471282959, 7.031954288482666, 7.417680740356445, 6.99337100982666, 7.202299118041992, 7.405534267425537, 6.963484764099121, 7.071299076080322, 7.202553749084473, 7.121299743652344, 7.113229751586914, 7.233953952789307, 7.171547889709473, 7.074029922485352, 6.886327266693115, 7.22685432434082, 6.599013328552246, 6.9451904296875, 6.938242435455322, 7.133424282073975, 7.1569037437438965, 7.027259826660156, 6.879076957702637, 6.84052848815918, 6.869926929473877, 7.156200408935547, 7.161357879638672, 7.119139671325684, 6.961492538452148, 7.020671844482422, 7.404061317443848, 6.8662638664245605, 6.834813117980957, 6.9740471839904785, 7.325422763824463, 6.80430793762207], "val_loss": [7.567975044250488, 7.446283340454102, 7.361580848693848, 7.295380592346191, 6.9730939865112305], "val_pp": [1935.2077298538604, 1713.474281203508, 1574.3155681500446, 1473.4703577673076, 1067.5155332391428], "val_acc": [0.051807403564453125, 0.052227020263671875, 0.049957275390625, 0.05741691589355469, 0.10369110107421875], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 7.567975044250488, "val_perplexity": 1935.2077298538604, "val_accuracy": 0.051807403564453125, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 7.446283340454102, "val_perplexity": 1713.474281203508, "val_accuracy": 0.052227020263671875, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 7.361580848693848, "val_perplexity": 1574.3155681500446, "val_accuracy": 0.049957275390625, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 7.295380592346191, "val_perplexity": 1473.4703577673076, "val_accuracy": 0.05741691589355469, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 6.9730939865112305, "val_perplexity": 1067.5155332391428, "val_accuracy": 0.10369110107421875, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.268243609, 0.268675392, 0.268387742, 0.268993562, 0.268876023, 0.269674513, 0.270229923, 0.267432868, 0.267361956, 0.267823464, 0.267259494, 0.274139197, 0.267863989, 0.274316899, 0.26808845, 0.26767367, 0.266328915, 0.266505846, 0.266523448, 0.268175886, 0.268076377, 0.266526867, 0.26703806, 0.266331833, 0.266377473, 0.266093056, 0.267748887, 0.268848322, 0.267253948, 0.267266058, 0.267557966, 0.268156461, 0.268441527, 0.267193971, 0.267206211, 0.267109267, 0.266929158, 0.267663081, 0.266275549, 0.266365305, 0.267159056, 0.265858629, 0.267576468, 0.268190909, 0.265889372, 0.267693118, 0.267114854, 0.268296824, 0.266539853, 0.265987128, 0.267217848, 0.268452212, 0.266696109, 0.267324371, 0.261436136, 0.266661726, 0.26778572, 0.268065668, 0.267969272, 0.267059442, 0.26906315, 0.268088688, 0.266139398, 0.265714274, 0.265965741, 0.266892776, 0.268642842, 0.26777281, 0.267849607, 0.26666901, 0.266934114, 0.267019481, 0.267092549, 0.266859331, 0.267140285, 0.26625136, 0.267169871, 0.267415006, 0.265411773, 0.267045582, 0.265771256], "train_time_seconds": 1095.110613765004, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 245121.77366002832, "peak_memory_bytes": 6039715328, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.09, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456/run.log new file mode 100644 index 0000000..a9880cb --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456/run.log @@ -0,0 +1,145 @@ +Starting Experiment: sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'signsgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Signum ( +Parameter Group 0 + dampening: 0.0 + lr: 0.0003 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0.0 + lr: 0.0003 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.539 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.728 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.583 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.670 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.571 iter_dt=2.71e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.492 val_pp=659.833 val_acc=0.152500 +Train: Iter=300 (0.036622 effective passes) train_loss=6.601 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=350 (0.042725 effective passes) train_loss=6.515 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=400 (0.048829 effective passes) train_loss=6.368 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=450 (0.054932 effective passes) train_loss=6.171 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=500 (0.061036 effective passes) train_loss=6.097 iter_dt=2.71e-01s lr=3.00e-04 +>Eval: Iter=512 (0.062501 effective passes) val_loss=6.202 val_pp=493.816 val_acc=0.169327 +Train: Iter=550 (0.067140 effective passes) train_loss=6.396 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=600 (0.073243 effective passes) train_loss=5.762 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=650 (0.079347 effective passes) train_loss=6.318 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=700 (0.085451 effective passes) train_loss=6.025 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=750 (0.091554 effective passes) train_loss=6.045 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=800 (0.097658 effective passes) train_loss=5.925 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=850 (0.103761 effective passes) train_loss=5.900 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=900 (0.109865 effective passes) train_loss=5.872 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=950 (0.115969 effective passes) train_loss=5.782 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=1000 (0.122072 effective passes) train_loss=5.772 iter_dt=2.70e-01s lr=3.00e-04 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=5.865 val_pp=352.379 val_acc=0.191597 +Train: Iter=1050 (0.128176 effective passes) train_loss=5.953 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=1100 (0.134279 effective passes) train_loss=5.856 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=1150 (0.140383 effective passes) train_loss=5.686 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=1200 (0.146487 effective passes) train_loss=5.683 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=1250 (0.152590 effective passes) train_loss=5.705 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=1300 (0.158694 effective passes) train_loss=5.776 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1350 (0.164797 effective passes) train_loss=5.659 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=1400 (0.170901 effective passes) train_loss=5.649 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=1450 (0.177005 effective passes) train_loss=5.544 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=1500 (0.183108 effective passes) train_loss=5.745 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=1550 (0.189212 effective passes) train_loss=5.534 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=1600 (0.195315 effective passes) train_loss=5.869 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=1650 (0.201419 effective passes) train_loss=5.406 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=1700 (0.207523 effective passes) train_loss=5.875 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=1750 (0.213626 effective passes) train_loss=5.541 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=1800 (0.219730 effective passes) train_loss=5.507 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=1850 (0.225834 effective passes) train_loss=5.714 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=1900 (0.231937 effective passes) train_loss=5.422 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=1950 (0.238041 effective passes) train_loss=5.586 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2000 (0.244144 effective passes) train_loss=5.616 iter_dt=2.71e-01s lr=3.00e-04 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=5.576 val_pp=263.959 val_acc=0.212559 +Train: Iter=2050 (0.250248 effective passes) train_loss=5.729 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=2100 (0.256352 effective passes) train_loss=5.579 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=2150 (0.262455 effective passes) train_loss=5.583 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=2200 (0.268559 effective passes) train_loss=5.370 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=2250 (0.274662 effective passes) train_loss=5.488 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=2300 (0.280766 effective passes) train_loss=5.476 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=2350 (0.286870 effective passes) train_loss=5.334 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=2400 (0.292973 effective passes) train_loss=5.752 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=2450 (0.299077 effective passes) train_loss=5.321 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=2500 (0.305180 effective passes) train_loss=5.643 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=2550 (0.311284 effective passes) train_loss=5.698 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=2600 (0.317388 effective passes) train_loss=5.170 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=2650 (0.323491 effective passes) train_loss=5.424 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=2700 (0.329595 effective passes) train_loss=5.584 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=2750 (0.335698 effective passes) train_loss=5.541 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2800 (0.341802 effective passes) train_loss=5.367 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2850 (0.347906 effective passes) train_loss=5.577 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=2900 (0.354009 effective passes) train_loss=5.577 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=2950 (0.360113 effective passes) train_loss=5.375 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3000 (0.366217 effective passes) train_loss=5.248 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3050 (0.372320 effective passes) train_loss=5.492 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3100 (0.378424 effective passes) train_loss=4.825 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3150 (0.384527 effective passes) train_loss=5.288 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3200 (0.390631 effective passes) train_loss=5.358 iter_dt=2.91e-01s lr=3.00e-04 +Train: Iter=3250 (0.396735 effective passes) train_loss=5.480 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3300 (0.402838 effective passes) train_loss=5.455 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3350 (0.408942 effective passes) train_loss=5.400 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3400 (0.415045 effective passes) train_loss=5.111 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3450 (0.421149 effective passes) train_loss=5.131 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3500 (0.427253 effective passes) train_loss=5.249 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3550 (0.433356 effective passes) train_loss=5.417 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=3600 (0.439460 effective passes) train_loss=5.539 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3650 (0.445563 effective passes) train_loss=5.457 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3700 (0.451667 effective passes) train_loss=5.238 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3750 (0.457771 effective passes) train_loss=5.355 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3800 (0.463874 effective passes) train_loss=5.727 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3850 (0.469978 effective passes) train_loss=5.156 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=3900 (0.476081 effective passes) train_loss=5.165 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=3950 (0.482185 effective passes) train_loss=5.321 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=4000 (0.488289 effective passes) train_loss=5.578 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=4050 (0.494392 effective passes) train_loss=5.169 iter_dt=2.74e-01s lr=3.00e-04 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=5.292 val_pp=198.721 val_acc=0.240356 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456/summary.json new file mode 100644 index 0000000..df9a868 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.539073467254639, 6.728296756744385, 6.5833516120910645, 6.669707298278809, 6.570851802825928, 6.601187229156494, 6.515040397644043, 6.367715358734131, 6.171159744262695, 6.0965776443481445, 6.396376609802246, 5.761934280395508, 6.317724227905273, 6.024676322937012, 6.044569969177246, 5.925426483154297, 5.899970054626465, 5.872091293334961, 5.781612396240234, 5.772290229797363, 5.952852249145508, 5.855764865875244, 5.686208724975586, 5.683112621307373, 5.704857349395752, 5.776449680328369, 5.659013748168945, 5.648622035980225, 5.543869972229004, 5.745351314544678, 5.534029006958008, 5.869158744812012, 5.405946731567383, 5.8745269775390625, 5.541051864624023, 5.507060527801514, 5.714026927947998, 5.42227840423584, 5.586086273193359, 5.616227626800537, 5.729321479797363, 5.5790205001831055, 5.58256721496582, 5.370292663574219, 5.488471031188965, 5.475547790527344, 5.333683013916016, 5.751682281494141, 5.3213396072387695, 5.643321990966797, 5.69836950302124, 5.169840335845947, 5.424440383911133, 5.584052085876465, 5.540617942810059, 5.367409706115723, 5.577015399932861, 5.576626300811768, 5.375210285186768, 5.248136520385742, 5.492259979248047, 4.825435638427734, 5.287872314453125, 5.358339309692383, 5.480490207672119, 5.454784393310547, 5.400092124938965, 5.110791206359863, 5.130974769592285, 5.248574256896973, 5.4169535636901855, 5.539371490478516, 5.457087516784668, 5.237912178039551, 5.354602813720703, 5.7274065017700195, 5.155779838562012, 5.165215969085693, 5.321410179138184, 5.577873229980469, 5.168817043304443], "val_loss": [6.4919915199279785, 6.202166557312012, 5.864711761474609, 5.575797080993652, 5.291903495788574], "val_pp": [659.8332509822862, 493.8157072855017, 352.3791817033458, 263.95887920271304, 198.7206234711055], "val_acc": [0.15250015258789062, 0.1693267822265625, 0.19159698486328125, 0.21255874633789062, 0.2403564453125], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.4919915199279785, "val_perplexity": 659.8332509822862, "val_accuracy": 0.15250015258789062, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 6.202166557312012, "val_perplexity": 493.8157072855017, "val_accuracy": 0.1693267822265625, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 5.864711761474609, "val_perplexity": 352.3791817033458, "val_accuracy": 0.19159698486328125, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 5.575797080993652, "val_perplexity": 263.95887920271304, "val_accuracy": 0.21255874633789062, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 5.291903495788574, "val_perplexity": 198.7206234711055, "val_accuracy": 0.2403564453125, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.270597784, 0.26911233, 0.269809498, 0.269610274, 0.270813423, 0.270478323, 0.269216888, 0.268988748, 0.269640079, 0.27067598, 0.271421787, 0.270223352, 0.270049855, 0.270504146, 0.270696613, 0.271280971, 0.268772155, 0.269810658, 0.270684628, 0.270390814, 0.269892728, 0.271391254, 0.271233008, 0.270117718, 0.270899386, 0.272608728, 0.269352638, 0.27091508, 0.270922965, 0.269987268, 0.269456115, 0.269616449, 0.270849003, 0.269862502, 0.272733062, 0.270438427, 0.269962503, 0.271958821, 0.272903478, 0.270633456, 0.269146872, 0.269965757, 0.270306614, 0.269235933, 0.269878377, 0.269563519, 0.268999327, 0.270464998, 0.270324791, 0.270561351, 0.270871206, 0.27032499, 0.269796524, 0.27211148, 0.272784337, 0.272756489, 0.273618813, 0.273000962, 0.272738228, 0.273015324, 0.273200971, 0.273547659, 0.272899655, 0.290598253, 0.273943249, 0.273915198, 0.272832879, 0.271590189, 0.273026551, 0.272034067, 0.271967884, 0.273374344, 0.273642211, 0.272854704, 0.273176714, 0.273036066, 0.274247915, 0.273391663, 0.273723384, 0.272974687, 0.274185128], "train_time_seconds": 1112.2371516669982, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 241347.32021644345, "peak_memory_bytes": 5909968384, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "signsgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456/run.log new file mode 100644 index 0000000..a09ee86 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456/run.log @@ -0,0 +1,145 @@ +Starting Experiment: sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'signum', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Signum ( +Parameter Group 0 + dampening: 0.0 + lr: 0.001 + momentum: 0.9 + nesterov: False + sign_update: True + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0.0 + lr: 0.001 + momentum: 0.9 + nesterov: False + sign_update: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.888 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.360 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.706 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.528 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.291 iter_dt=2.70e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.178 val_pp=481.885 val_acc=0.151791 +Train: Iter=300 (0.036622 effective passes) train_loss=5.848 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=6.034 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=5.696 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=5.514 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=5.361 iter_dt=2.73e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.435 val_pp=229.296 val_acc=0.195808 +Train: Iter=550 (0.067140 effective passes) train_loss=5.522 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.996 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=5.424 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=5.181 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=5.153 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.975 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.886 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.811 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.867 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.858 iter_dt=2.76e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.796 val_pp=121.082 val_acc=0.251291 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.890 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.709 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.704 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.625 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.643 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.693 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.681 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.686 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.541 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.655 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.543 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.839 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.401 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.746 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.412 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.430 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.736 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.339 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.308 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.489 iter_dt=2.75e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.458 val_pp=86.295 val_acc=0.278936 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.503 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.465 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.439 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.424 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.454 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.191 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.361 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.377 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.354 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.634 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.406 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.171 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.571 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.464 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.375 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.219 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.516 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.446 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.371 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.215 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.382 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.993 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.254 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.333 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.408 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.283 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.448 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.179 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.162 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.267 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.330 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.422 iter_dt=2.65e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.344 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.222 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.223 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.333 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.136 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.243 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.282 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.352 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.229 iter_dt=2.72e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.240 val_pp=69.406 val_acc=0.297480 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456/summary.json new file mode 100644 index 0000000..e578268 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.888364791870117, 7.359785079956055, 6.705845355987549, 6.528271675109863, 6.290552139282227, 5.848129749298096, 6.033835411071777, 5.696138381958008, 5.514348030090332, 5.361051082611084, 5.521885871887207, 4.995760440826416, 5.4243879318237305, 5.181363105773926, 5.15294885635376, 4.974851608276367, 4.885901927947998, 4.811273574829102, 4.866910934448242, 4.85810661315918, 4.890223503112793, 4.709296703338623, 4.704229354858398, 4.624917030334473, 4.643379211425781, 4.69318962097168, 4.680987358093262, 4.68607759475708, 4.5408124923706055, 4.654655933380127, 4.543033599853516, 4.838827610015869, 4.401403427124023, 4.745964050292969, 4.412299156188965, 4.430261611938477, 4.735792636871338, 4.338796615600586, 4.307640552520752, 4.489067077636719, 4.503174781799316, 4.465493202209473, 4.438967704772949, 4.424434661865234, 4.453872203826904, 4.191291809082031, 4.3610944747924805, 4.37667179107666, 4.354365825653076, 4.633957862854004, 4.405974864959717, 4.171074867248535, 4.570606708526611, 4.4641923904418945, 4.3751749992370605, 4.2193803787231445, 4.515949249267578, 4.446409225463867, 4.370535850524902, 4.214850902557373, 4.381504058837891, 3.9928436279296875, 4.254465103149414, 4.332881927490234, 4.4080491065979, 4.283298015594482, 4.447696208953857, 4.179020881652832, 4.162123203277588, 4.267441272735596, 4.329947471618652, 4.422183990478516, 4.3441162109375, 4.221916198730469, 4.222599983215332, 4.333066463470459, 4.136030673980713, 4.242742538452148, 4.2822418212890625, 4.352477073669434, 4.228793621063232], "val_loss": [6.17771053314209, 5.435019493103027, 4.796474933624268, 4.4577717781066895, 4.239973068237305], "val_pp": [481.8854246932734, 229.2964796663352, 121.08244863877887, 86.29475152175984, 69.4057846405103], "val_acc": [0.15179061889648438, 0.19580841064453125, 0.25129127502441406, 0.27893638610839844, 0.29747962951660156], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.17771053314209, "val_perplexity": 481.8854246932734, "val_accuracy": 0.15179061889648438, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 5.435019493103027, "val_perplexity": 229.2964796663352, "val_accuracy": 0.19580841064453125, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.796474933624268, "val_perplexity": 121.08244863877887, "val_accuracy": 0.25129127502441406, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.4577717781066895, "val_perplexity": 86.29475152175984, "val_accuracy": 0.27893638610839844, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.239973068237305, "val_perplexity": 69.4057846405103, "val_accuracy": 0.29747962951660156, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.273668818, 0.27385819, 0.271651212, 0.271580048, 0.270279874, 0.272842739, 0.273233434, 0.272601139, 0.272154836, 0.273320914, 0.275518575, 0.275322139, 0.273667947, 0.273462886, 0.272491164, 0.279843696, 0.272865027, 0.272885848, 0.271898495, 0.275871793, 0.273507555, 0.274142541, 0.273596846, 0.273610226, 0.272241045, 0.272589592, 0.272717601, 0.272329012, 0.272277146, 0.270316559, 0.275905889, 0.270793949, 0.275100452, 0.274596061, 0.275550056, 0.27453328, 0.2726782, 0.273441649, 0.270509425, 0.274747377, 0.273656229, 0.270055977, 0.270075172, 0.272262362, 0.2738365, 0.273125824, 0.272096094, 0.274257934, 0.27526534, 0.273948246, 0.275286388, 0.272751628, 0.272476626, 0.274144076, 0.275263985, 0.272439425, 0.272805078, 0.273184644, 0.273507162, 0.273334647, 0.27249938, 0.272425476, 0.274739074, 0.275154555, 0.274006484, 0.274495272, 0.273074654, 0.272534801, 0.272026407, 0.274041822, 0.274274103, 0.265055793, 0.272981249, 0.272468643, 0.274280719, 0.272614923, 0.272446845, 0.273019279, 0.274226964, 0.272156062, 0.272268959], "train_time_seconds": 1119.0858592749996, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 239870.29571967432, "peak_memory_bytes": 6040567296, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "signum", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456/run.log new file mode 100644 index 0000000..4905125 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456/run.log @@ -0,0 +1,155 @@ +Starting Experiment: sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'soap', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SOAP ( +Parameter Group 0 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.001 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.001 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.365 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.134 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=5.701 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.556 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.445 iter_dt=2.84e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.292 val_pp=198.774 val_acc=0.209633 +Train: Iter=300 (0.036622 effective passes) train_loss=5.092 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=350 (0.042725 effective passes) train_loss=5.259 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=400 (0.048829 effective passes) train_loss=4.864 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=450 (0.054932 effective passes) train_loss=4.606 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=500 (0.061036 effective passes) train_loss=4.637 iter_dt=2.81e-01s lr=1.00e-03 +>Eval: Iter=512 (0.062501 effective passes) val_loss=4.701 val_pp=110.016 val_acc=0.257669 +Train: Iter=550 (0.067140 effective passes) train_loss=4.668 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=600 (0.073243 effective passes) train_loss=4.415 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=650 (0.079347 effective passes) train_loss=4.755 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=700 (0.085451 effective passes) train_loss=4.643 iter_dt=2.77e-01s lr=1.00e-03 +Train: Iter=750 (0.091554 effective passes) train_loss=4.660 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=800 (0.097658 effective passes) train_loss=4.478 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=850 (0.103761 effective passes) train_loss=4.418 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=900 (0.109865 effective passes) train_loss=4.371 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=950 (0.115969 effective passes) train_loss=4.452 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.494 iter_dt=2.82e-01s lr=1.00e-03 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.373 val_pp=79.320 val_acc=0.284046 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.506 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.318 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.338 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.262 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.295 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.338 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.358 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.374 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.241 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.309 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.255 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.525 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.103 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.433 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.118 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.144 iter_dt=2.87e-01s lr=1.00e-03 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.451 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.053 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=1950 (0.238041 effective passes) train_loss=3.993 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.224 iter_dt=2.82e-01s lr=1.00e-03 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.174 val_pp=64.982 val_acc=0.300755 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.174 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.193 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.164 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.149 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.203 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=2300 (0.280766 effective passes) train_loss=3.913 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.112 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.101 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.123 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.383 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.144 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2600 (0.317388 effective passes) train_loss=3.945 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.371 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.230 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.131 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=2800 (0.341802 effective passes) train_loss=3.983 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.293 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.194 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.150 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3000 (0.366217 effective passes) train_loss=3.986 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.172 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.790 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.022 iter_dt=2.86e-01s lr=1.00e-03 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.104 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.172 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.030 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.253 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=3400 (0.415045 effective passes) train_loss=3.979 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=3450 (0.421149 effective passes) train_loss=3.940 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.057 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.107 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.182 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.122 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.013 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3750 (0.457771 effective passes) train_loss=3.993 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.094 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=3850 (0.469978 effective passes) train_loss=3.930 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.049 iter_dt=2.86e-01s lr=1.00e-03 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.102 iter_dt=2.89e-01s lr=1.00e-03 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.133 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.037 iter_dt=2.83e-01s lr=1.00e-03 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.039 val_pp=56.759 val_acc=0.311636 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456/summary.json new file mode 100644 index 0000000..06fe7a3 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.365467071533203, 6.1341142654418945, 5.70082950592041, 5.556131839752197, 5.445219039916992, 5.092162132263184, 5.259283065795898, 4.864073753356934, 4.6055827140808105, 4.6374311447143555, 4.667691707611084, 4.414956092834473, 4.755376815795898, 4.643250465393066, 4.659609794616699, 4.477742671966553, 4.418200492858887, 4.3706512451171875, 4.4515838623046875, 4.493964195251465, 4.505744934082031, 4.3176374435424805, 4.337692737579346, 4.262277603149414, 4.295166969299316, 4.337975025177002, 4.358151435852051, 4.374484539031982, 4.240650653839111, 4.309340476989746, 4.254825592041016, 4.52507209777832, 4.103074550628662, 4.432656288146973, 4.118073463439941, 4.144458293914795, 4.450549125671387, 4.052957534790039, 3.9926018714904785, 4.224133491516113, 4.174249649047852, 4.193161964416504, 4.164039611816406, 4.149186611175537, 4.203186988830566, 3.912928819656372, 4.112483978271484, 4.101293563842773, 4.123102188110352, 4.382527828216553, 4.14388370513916, 3.944678783416748, 4.3705315589904785, 4.230413436889648, 4.13134765625, 3.9834513664245605, 4.293128490447998, 4.193803310394287, 4.1504926681518555, 3.9856114387512207, 4.172029495239258, 3.789522647857666, 4.022111892700195, 4.10445499420166, 4.171809196472168, 4.030104637145996, 4.252615928649902, 3.979337692260742, 3.9398961067199707, 4.056578159332275, 4.10749626159668, 4.181510925292969, 4.122231960296631, 4.012929916381836, 3.9929332733154297, 4.093770980834961, 3.9304392337799072, 4.0486884117126465, 4.102168083190918, 4.133332252502441, 4.037290573120117], "val_loss": [5.292172431945801, 4.70062780380249, 4.373493194580078, 4.17411470413208, 4.038823127746582], "val_pp": [198.77407378302857, 110.0158715181136, 79.31999596153322, 64.98210318721175, 56.75935059719494], "val_acc": [0.20963287353515625, 0.25766944885253906, 0.2840461730957031, 0.3007545471191406, 0.31163597106933594], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.292172431945801, "val_perplexity": 198.77407378302857, "val_accuracy": 0.20963287353515625, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 4.70062780380249, "val_perplexity": 110.0158715181136, "val_accuracy": 0.25766944885253906, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.373493194580078, "val_perplexity": 79.31999596153322, "val_accuracy": 0.2840461730957031, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.17411470413208, "val_perplexity": 64.98210318721175, "val_accuracy": 0.3007545471191406, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.038823127746582, "val_perplexity": 56.75935059719494, "val_accuracy": 0.31163597106933594, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.276764206, 0.280707963, 0.280550637, 0.281093567, 0.283680571, 0.284873341, 0.281305943, 0.283483327, 0.282109717, 0.281114468, 0.282569762, 0.279961714, 0.282511217, 0.276759444, 0.281153954, 0.281267181, 0.283151106, 0.28316, 0.283673498, 0.28200925, 0.281159337, 0.281084235, 0.284677807, 0.282214634, 0.28307791, 0.283284536, 0.281998175, 0.282875367, 0.280852494, 0.282175329, 0.27991479, 0.282845616, 0.282571292, 0.280677341, 0.281094423, 0.286574676, 0.285140489, 0.282609966, 0.281630736, 0.281598949, 0.282393729, 0.281891271, 0.281713838, 0.284989993, 0.281511693, 0.284000402, 0.281438556, 0.280913716, 0.281405212, 0.28122375, 0.281290239, 0.276384305, 0.280990906, 0.282040469, 0.281621874, 0.28063221, 0.280892263, 0.281541334, 0.281813823, 0.282289893, 0.281121556, 0.28267198, 0.285883941, 0.281207287, 0.28043993, 0.28099963, 0.284010674, 0.278666082, 0.280433283, 0.282510645, 0.281504107, 0.281161124, 0.28236191, 0.281453793, 0.2848386, 0.28111033, 0.282120538, 0.286342674, 0.289408857, 0.281242752, 0.283035785], "train_time_seconds": 1284.3550443230042, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 209004.0890067862, "peak_memory_bytes": 6559245312, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "soap", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456/run.log b/exps/sub_one_pass/formal/sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456/run.log new file mode 100644 index 0000000..34191b0 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456/run.log @@ -0,0 +1,145 @@ +Starting Experiment: sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 4096, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 268435456, 'eval_at_tokens': [16777216, 33554432, 67108864, 134217728, 268435456], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sophiag', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 268,435,456 tokens, 4,096 optimizer steps, 0.500008 effective passes +Evaluation token boundaries: 16,777,216, 33,554,432, 67,108,864, 134,217,728, 268,435,456 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SophiaG ( +Parameter Group 0 + betas: (0.9, 0.999) + capturable: False + lr: 0.0003 + maximize: False + rho: 0.04 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + capturable: False + lr: 0.0003 + maximize: False + rho: 0.04 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.758 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.832 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.463 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.363 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.193 iter_dt=3.23e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.090 val_pp=441.610 val_acc=0.157600 +Train: Iter=300 (0.036622 effective passes) train_loss=5.865 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=350 (0.042725 effective passes) train_loss=5.999 iter_dt=3.22e-01s lr=3.00e-04 +Train: Iter=400 (0.048829 effective passes) train_loss=5.706 iter_dt=3.25e-01s lr=3.00e-04 +Train: Iter=450 (0.054932 effective passes) train_loss=5.501 iter_dt=3.27e-01s lr=3.00e-04 +Train: Iter=500 (0.061036 effective passes) train_loss=5.394 iter_dt=3.20e-01s lr=3.00e-04 +>Eval: Iter=512 (0.062501 effective passes) val_loss=5.460 val_pp=235.157 val_acc=0.195499 +Train: Iter=550 (0.067140 effective passes) train_loss=5.557 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=600 (0.073243 effective passes) train_loss=5.029 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=650 (0.079347 effective passes) train_loss=5.469 iter_dt=3.21e-01s lr=3.00e-04 +Train: Iter=700 (0.085451 effective passes) train_loss=5.242 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=750 (0.091554 effective passes) train_loss=5.187 iter_dt=3.22e-01s lr=3.00e-04 +Train: Iter=800 (0.097658 effective passes) train_loss=5.003 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=850 (0.103761 effective passes) train_loss=4.944 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=900 (0.109865 effective passes) train_loss=4.851 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=950 (0.115969 effective passes) train_loss=4.867 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1000 (0.122072 effective passes) train_loss=4.880 iter_dt=3.19e-01s lr=3.00e-04 +>Eval: Iter=1024 (0.125002 effective passes) val_loss=4.813 val_pp=123.039 val_acc=0.248173 +Train: Iter=1050 (0.128176 effective passes) train_loss=4.919 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=1100 (0.134279 effective passes) train_loss=4.722 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=1150 (0.140383 effective passes) train_loss=4.695 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1200 (0.146487 effective passes) train_loss=4.631 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1250 (0.152590 effective passes) train_loss=4.640 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1300 (0.158694 effective passes) train_loss=4.688 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=1350 (0.164797 effective passes) train_loss=4.674 iter_dt=3.21e-01s lr=3.00e-04 +Train: Iter=1400 (0.170901 effective passes) train_loss=4.669 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1450 (0.177005 effective passes) train_loss=4.533 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1500 (0.183108 effective passes) train_loss=4.651 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1550 (0.189212 effective passes) train_loss=4.531 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1600 (0.195315 effective passes) train_loss=4.817 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=1650 (0.201419 effective passes) train_loss=4.382 iter_dt=3.21e-01s lr=3.00e-04 +Train: Iter=1700 (0.207523 effective passes) train_loss=4.727 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=1750 (0.213626 effective passes) train_loss=4.385 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1800 (0.219730 effective passes) train_loss=4.418 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1850 (0.225834 effective passes) train_loss=4.718 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1900 (0.231937 effective passes) train_loss=4.302 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=1950 (0.238041 effective passes) train_loss=4.304 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2000 (0.244144 effective passes) train_loss=4.465 iter_dt=3.20e-01s lr=3.00e-04 +>Eval: Iter=2048 (0.250004 effective passes) val_loss=4.431 val_pp=83.999 val_acc=0.281746 +Train: Iter=2050 (0.250248 effective passes) train_loss=4.483 iter_dt=3.23e-01s lr=3.00e-04 +Train: Iter=2100 (0.256352 effective passes) train_loss=4.440 iter_dt=3.24e-01s lr=3.00e-04 +Train: Iter=2150 (0.262455 effective passes) train_loss=4.422 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2200 (0.268559 effective passes) train_loss=4.381 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=2250 (0.274662 effective passes) train_loss=4.438 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2300 (0.280766 effective passes) train_loss=4.153 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=2350 (0.286870 effective passes) train_loss=4.330 iter_dt=3.29e-01s lr=3.00e-04 +Train: Iter=2400 (0.292973 effective passes) train_loss=4.341 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=2450 (0.299077 effective passes) train_loss=4.315 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2500 (0.305180 effective passes) train_loss=4.579 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=2550 (0.311284 effective passes) train_loss=4.383 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2600 (0.317388 effective passes) train_loss=4.137 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2650 (0.323491 effective passes) train_loss=4.554 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=2700 (0.329595 effective passes) train_loss=4.436 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2750 (0.335698 effective passes) train_loss=4.342 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2800 (0.341802 effective passes) train_loss=4.181 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2850 (0.347906 effective passes) train_loss=4.485 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2900 (0.354009 effective passes) train_loss=4.411 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=2950 (0.360113 effective passes) train_loss=4.345 iter_dt=3.28e-01s lr=3.00e-04 +Train: Iter=3000 (0.366217 effective passes) train_loss=4.192 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=3050 (0.372320 effective passes) train_loss=4.365 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=3100 (0.378424 effective passes) train_loss=3.958 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=3150 (0.384527 effective passes) train_loss=4.230 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=3200 (0.390631 effective passes) train_loss=4.291 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=3250 (0.396735 effective passes) train_loss=4.367 iter_dt=3.18e-01s lr=3.00e-04 +Train: Iter=3300 (0.402838 effective passes) train_loss=4.249 iter_dt=3.21e-01s lr=3.00e-04 +Train: Iter=3350 (0.408942 effective passes) train_loss=4.427 iter_dt=3.22e-01s lr=3.00e-04 +Train: Iter=3400 (0.415045 effective passes) train_loss=4.154 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=3450 (0.421149 effective passes) train_loss=4.117 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=3500 (0.427253 effective passes) train_loss=4.222 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=3550 (0.433356 effective passes) train_loss=4.280 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=3600 (0.439460 effective passes) train_loss=4.384 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=3650 (0.445563 effective passes) train_loss=4.338 iter_dt=3.19e-01s lr=3.00e-04 +Train: Iter=3700 (0.451667 effective passes) train_loss=4.184 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=3750 (0.457771 effective passes) train_loss=4.193 iter_dt=3.21e-01s lr=3.00e-04 +Train: Iter=3800 (0.463874 effective passes) train_loss=4.309 iter_dt=3.24e-01s lr=3.00e-04 +Train: Iter=3850 (0.469978 effective passes) train_loss=4.105 iter_dt=3.25e-01s lr=3.00e-04 +Train: Iter=3900 (0.476081 effective passes) train_loss=4.204 iter_dt=3.29e-01s lr=3.00e-04 +Train: Iter=3950 (0.482185 effective passes) train_loss=4.273 iter_dt=3.26e-01s lr=3.00e-04 +Train: Iter=4000 (0.488289 effective passes) train_loss=4.323 iter_dt=3.21e-01s lr=3.00e-04 +Train: Iter=4050 (0.494392 effective passes) train_loss=4.197 iter_dt=3.19e-01s lr=3.00e-04 +>Eval: Iter=4096 (0.500008 effective passes) val_loss=4.210 val_pp=67.352 val_acc=0.299429 diff --git a/exps/sub_one_pass/formal/sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456/summary.json b/exps/sub_one_pass/formal/sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456/summary.json new file mode 100644 index 0000000..055335d --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.757791519165039, 6.831812858581543, 6.462698936462402, 6.362981796264648, 6.193485260009766, 5.865413188934326, 5.998810291290283, 5.706107139587402, 5.501117706298828, 5.394097805023193, 5.556639194488525, 5.029092311859131, 5.4689788818359375, 5.242239475250244, 5.187270164489746, 5.0025482177734375, 4.943995475769043, 4.851362228393555, 4.867425918579102, 4.879845142364502, 4.919020652770996, 4.722160339355469, 4.6945648193359375, 4.630609035491943, 4.6398725509643555, 4.687614440917969, 4.67423152923584, 4.668687343597412, 4.532627105712891, 4.651307582855225, 4.530592918395996, 4.81685733795166, 4.382007122039795, 4.727011680603027, 4.384518623352051, 4.418117523193359, 4.718041896820068, 4.30230188369751, 4.303709030151367, 4.465205192565918, 4.483181953430176, 4.440488815307617, 4.421927452087402, 4.381412982940674, 4.438258647918701, 4.152960300445557, 4.329588890075684, 4.341378211975098, 4.314942359924316, 4.578920841217041, 4.382826805114746, 4.137253761291504, 4.553671836853027, 4.435659885406494, 4.341514587402344, 4.181094169616699, 4.485236644744873, 4.411450386047363, 4.34455680847168, 4.192265510559082, 4.365353584289551, 3.9581542015075684, 4.2301554679870605, 4.291140556335449, 4.3673481941223145, 4.24853515625, 4.426924228668213, 4.154125213623047, 4.116988182067871, 4.221793174743652, 4.27977180480957, 4.384456634521484, 4.338150978088379, 4.184466361999512, 4.192899227142334, 4.309342384338379, 4.105001449584961, 4.203769683837891, 4.273124694824219, 4.323121070861816, 4.196615219116211], "val_loss": [6.090432167053223, 5.460256576538086, 4.812508583068848, 4.430809497833252, 4.209933280944824], "val_pp": [441.6104109948902, 235.15688888849053, 123.03948817490895, 83.99913636632618, 67.35185526652654], "val_acc": [0.15760040283203125, 0.19549942016601562, 0.24817276000976562, 0.28174591064453125, 0.29942893981933594], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.090432167053223, "val_perplexity": 441.6104109948902, "val_accuracy": 0.15760040283203125, "full_eval": false, "num_eval_batches": 64}, {"iter": 512, "tokens": 33554432, "data_exposure": 0.06250095368886854, "val_loss": 5.460256576538086, "val_perplexity": 235.15688888849053, "val_accuracy": 0.19549942016601562, "full_eval": false, "num_eval_batches": 64}, {"iter": 1024, "tokens": 67108864, "data_exposure": 0.1250019073777371, "val_loss": 4.812508583068848, "val_perplexity": 123.03948817490895, "val_accuracy": 0.24817276000976562, "full_eval": false, "num_eval_batches": 64}, {"iter": 2048, "tokens": 134217728, "data_exposure": 0.2500038147554742, "val_loss": 4.430809497833252, "val_perplexity": 83.99913636632618, "val_accuracy": 0.28174591064453125, "full_eval": false, "num_eval_batches": 64}, {"iter": 4096, "tokens": 268435456, "data_exposure": 0.5000076295109483, "val_loss": 4.209933280944824, "val_perplexity": 67.35185526652654, "val_accuracy": 0.29942893981933594, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.318591158, 0.319253209, 0.318332579, 0.319031692, 0.322724915, 0.319452034, 0.321894144, 0.324657186, 0.32687512, 0.320283361, 0.318368813, 0.318099413, 0.321117886, 0.318587282, 0.322110809, 0.318659635, 0.318879279, 0.319338251, 0.319462659, 0.318735191, 0.318489549, 0.318223139, 0.318736436, 0.319210125, 0.318803399, 0.319930734, 0.321282597, 0.31879873, 0.318735375, 0.319099409, 0.319325293, 0.320284256, 0.320664158, 0.3196266, 0.318530915, 0.318645461, 0.31891178, 0.318702426, 0.319226505, 0.320229634, 0.322960328, 0.323531491, 0.318848411, 0.318301335, 0.318769641, 0.320156345, 0.329210595, 0.319533468, 0.319372838, 0.318318464, 0.319014247, 0.319339634, 0.32042621, 0.318580566, 0.318569663, 0.318948693, 0.31925378, 0.319388833, 0.327800792, 0.320265742, 0.318001707, 0.318190846, 0.318700143, 0.318357459, 0.318147536, 0.320996124, 0.321832077, 0.320025643, 0.320021195, 0.319443598, 0.318949711, 0.318947214, 0.318630143, 0.319646767, 0.320972101, 0.323784693, 0.324867667, 0.329448484, 0.326201852, 0.321162932, 0.319032943], "train_time_seconds": 1145.1167670799987, "processed_tokens_this_run": 268435456, "mean_tokens_per_second": 234417.54039153538, "peak_memory_bytes": 14279713792, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 4096, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 4096, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 268435456, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sophiag", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256, 512, 1024, 2048, 4096], "token_budget_plan": {"train_token_budget": 268435456, "tokens_per_iteration": 65536, "iterations": 4096, "eval_at_tokens": [16777216, 33554432, 67108864, 134217728, 268435456], "eval_at_steps": [256, 512, 1024, 2048, 4096], "data_unique_tokens": 536862720, "target_data_exposure": 0.5000076295109483}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216/run.log new file mode 100644 index 0000000..7d74ae6 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216/run.log @@ -0,0 +1,71 @@ +Starting Experiment: sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adafactor', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Adafactor ( +Parameter Group 0 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.0001 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.1 + +Parameter Group 1 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.0001 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.464 iter_dt=2.87e-01s lr=1.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.467 iter_dt=2.89e-01s lr=1.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.176 iter_dt=2.90e-01s lr=1.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.141 iter_dt=2.87e-01s lr=1.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=5.998 iter_dt=2.87e-01s lr=1.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.911 val_pp=369.029 val_acc=0.181002 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216/summary.json new file mode 100644 index 0000000..17572af --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.464216709136963, 6.466541290283203, 6.176494598388672, 6.141362190246582, 5.997767448425293], "val_loss": [5.9108781814575195], "val_pp": [369.0286212845167], "val_acc": [0.1810016632080078], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.9108781814575195, "val_perplexity": 369.0286212845167, "val_accuracy": 0.1810016632080078, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.287013222, 0.288853177, 0.290263592, 0.286930643, 0.287364822], "train_time_seconds": 74.08217012899999, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 226467.66382228915, "peak_memory_bytes": 6040051712, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "adafactor", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..7b9560a --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,71 @@ +Starting Experiment: sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adafactor', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Adafactor ( +Parameter Group 0 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.0003 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.1 + +Parameter Group 1 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.0003 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.464 iter_dt=2.91e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.465 iter_dt=2.92e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.161 iter_dt=2.93e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.141 iter_dt=2.88e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=5.998 iter_dt=2.88e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.907 val_pp=367.467 val_acc=0.181429 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..49a5cbb --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.464322566986084, 6.4649152755737305, 6.161150932312012, 6.1413679122924805, 5.998466491699219], "val_loss": [5.906637191772461], "val_pp": [367.46688974311246], "val_acc": [0.1814289093017578], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.906637191772461, "val_perplexity": 367.46688974311246, "val_accuracy": 0.1814289093017578, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.290634363, 0.292200246, 0.293350968, 0.288047009, 0.287928754], "train_time_seconds": 74.959563125, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 223816.8860726001, "peak_memory_bytes": 6040051712, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "adafactor", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..c72703e --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,71 @@ +Starting Experiment: sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adafactor', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Adafactor ( +Parameter Group 0 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.001 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.1 + +Parameter Group 1 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.001 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.464 iter_dt=2.87e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.465 iter_dt=2.89e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.159 iter_dt=2.88e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.138 iter_dt=2.88e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.000 iter_dt=2.89e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.910 val_pp=368.706 val_acc=0.181269 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..9330583 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.464395999908447, 6.465332508087158, 6.158690452575684, 6.138473033905029, 5.999759674072266], "val_loss": [5.910003662109375], "val_pp": [368.7060399045575], "val_acc": [0.18126869201660156], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.910003662109375, "val_perplexity": 368.7060399045575, "val_accuracy": 0.18126869201660156, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.287350562, 0.288623245, 0.287929488, 0.288409694, 0.289218267], "train_time_seconds": 74.32661689599998, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 225722.85273625707, "peak_memory_bytes": 6040051712, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "adafactor", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..54fc660 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,71 @@ +Starting Experiment: sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adafactor', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Adafactor ( +Parameter Group 0 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.003 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.1 + +Parameter Group 1 + beta1: 0.9 + clip_threshold: 1.0 + decay_rate: -0.8 + eps2: (1e-30, 0.001) + lr: 0.003 + relative_step: True + scale_parameter: True + warmup_init: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.464 iter_dt=2.90e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.465 iter_dt=2.89e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.160 iter_dt=2.89e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.141 iter_dt=2.89e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.000 iter_dt=2.90e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.909 val_pp=368.475 val_acc=0.181158 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..bb75b8e --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.4642767906188965, 6.464921474456787, 6.159729480743408, 6.141401290893555, 6.000347137451172], "val_loss": [5.909377574920654], "val_pp": [368.4752701803194], "val_acc": [0.18115806579589844], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.909377574920654, "val_perplexity": 368.4752701803194, "val_accuracy": 0.18115806579589844, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.289562451, 0.28878486, 0.288632373, 0.289392575, 0.289943051], "train_time_seconds": 74.47961303800007, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 225259.1724857665, "peak_memory_bytes": 6040051712, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "adafactor", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..cd97d48 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,76 @@ +Starting Experiment: sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +using fused AdamW: True + +Optimizer: +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.0003 + maximize: False + weight_decay: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.0003 + maximize: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.330 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.384 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.082 iter_dt=2.72e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.069 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=5.924 iter_dt=2.71e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.826 val_pp=338.991 val_acc=0.187662 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..6d46164 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.329633712768555, 6.383791446685791, 6.08198356628418, 6.069149971008301, 5.924485206604004], "val_loss": [5.825977802276611], "val_pp": [338.9911101928887], "val_acc": [0.18766212463378906], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.825977802276611, "val_perplexity": 338.9911101928887, "val_accuracy": 0.18766212463378906, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.269803999, 0.269872918, 0.271731558, 0.272719428, 0.271370649], "train_time_seconds": 70.02755577000005, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 239580.2026148597, "peak_memory_bytes": 6175430656, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "adamw", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..9266294 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,76 @@ +Starting Experiment: sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +using fused AdamW: True + +Optimizer: +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.001 + maximize: False + weight_decay: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.001 + maximize: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.346 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.392 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.058 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.896 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.759 iter_dt=2.75e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.634 val_pp=279.670 val_acc=0.185808 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..3ec61e0 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.346497535705566, 6.392087459564209, 6.05785608291626, 5.896170616149902, 5.759103298187256], "val_loss": [5.633615016937256], "val_pp": [279.67024912779885], "val_acc": [0.1858081817626953], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.633615016937256, "val_perplexity": 279.67024912779885, "val_accuracy": 0.1858081817626953, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.273266617, 0.274891578, 0.275064938, 0.274786424, 0.275345401], "train_time_seconds": 70.83568091299999, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 236846.9644642181, "peak_memory_bytes": 6175430656, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "adamw", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..e83890f --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,76 @@ +Starting Experiment: sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +using fused AdamW: True + +Optimizer: +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.003 + maximize: False + weight_decay: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.8, 0.999) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-08 + foreach: None + fused: True + lr: 0.003 + maximize: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.736 iter_dt=2.74e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.974 iter_dt=2.68e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.761 iter_dt=2.70e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.556 iter_dt=2.71e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.427 iter_dt=2.80e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.316 val_pp=553.568 val_acc=0.140190 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..4afe74b --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.735668182373047, 6.974024295806885, 6.7613115310668945, 6.556407451629639, 6.427488327026367], "val_loss": [6.316389083862305], "val_pp": [553.5681306207249], "val_acc": [0.14019012451171875], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.316389083862305, "val_perplexity": 553.5681306207249, "val_accuracy": 0.14019012451171875, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.274152891, 0.268291234, 0.2702828, 0.270791267, 0.280442317], "train_time_seconds": 69.917800776, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 239956.28886769776, "peak_memory_bytes": 6175430656, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "adamw", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..256e848 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,67 @@ +Starting Experiment: sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'ademamix', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.999, 'adema_alpha': 8.0, 'adema_beta3_warmup': 4096, 'adema_alpha_warmup': 4096, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdEMAMix ( +Parameter Group 0 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.0003 + weight_decay: 0.1 + +Parameter Group 1 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.0003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.476 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.474 iter_dt=2.77e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.130 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.066 iter_dt=2.81e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=5.898 iter_dt=2.80e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.787 val_pp=326.190 val_acc=0.186926 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..0c6822a --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.4755330085754395, 6.4738311767578125, 6.130029678344727, 6.066319942474365, 5.897876262664795], "val_loss": [5.787484645843506], "val_pp": [326.1902345954303], "val_acc": [0.18692588806152344], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.787484645843506, "val_perplexity": 326.1902345954303, "val_accuracy": 0.18692588806152344, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.284236159, 0.277330369, 0.284183462, 0.280842308, 0.280397326], "train_time_seconds": 72.67416188899998, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 230855.30763498787, "peak_memory_bytes": 6314994176, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "ademamix", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.999, "adema_alpha": 8.0, "adema_beta3_warmup": 4096, "adema_alpha_warmup": 4096, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..9a010d3 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,67 @@ +Starting Experiment: sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'ademamix', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.999, 'adema_alpha': 8.0, 'adema_beta3_warmup': 4096, 'adema_alpha_warmup': 4096, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdEMAMix ( +Parameter Group 0 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.001 + weight_decay: 0.1 + +Parameter Group 1 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.001 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.349 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.449 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.112 iter_dt=2.79e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.960 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.798 iter_dt=2.78e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.665 val_pp=288.530 val_acc=0.183367 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..59f5520 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.348716735839844, 6.448815822601318, 6.111617565155029, 5.960325717926025, 5.797917366027832], "val_loss": [5.664802074432373], "val_pp": [288.5297684001515], "val_acc": [0.1833667755126953], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.664802074432373, "val_perplexity": 288.5297684001515, "val_accuracy": 0.1833667755126953, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.274329065, 0.279023283, 0.279304848, 0.280396769, 0.278401741], "train_time_seconds": 71.56953362500002, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 234418.40613223636, "peak_memory_bytes": 6314994176, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "ademamix", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.999, "adema_alpha": 8.0, "adema_beta3_warmup": 4096, "adema_alpha_warmup": 4096, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..9faed6f --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,67 @@ +Starting Experiment: sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'ademamix', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.999, 'adema_alpha': 8.0, 'adema_beta3_warmup': 4096, 'adema_alpha_warmup': 4096, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdEMAMix ( +Parameter Group 0 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.003 + weight_decay: 0.1 + +Parameter Group 1 + alpha: 8.0 + alpha_warmup: 4096 + beta3_warmup: 4096 + betas: (0.9, 0.999, 0.999) + eps: 1e-08 + lr: 0.003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.719 iter_dt=2.92e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.078 iter_dt=2.79e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.763 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.680 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.531 iter_dt=2.77e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.462 val_pp=640.196 val_acc=0.128218 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..223aaf7 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.718557357788086, 7.077777862548828, 6.762542724609375, 6.6795125007629395, 6.53051233291626], "val_loss": [6.46177864074707], "val_pp": [640.1959454203139], "val_acc": [0.1282176971435547], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.46177864074707, "val_perplexity": 640.1959454203139, "val_accuracy": 0.1282176971435547, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.291787181, 0.278923342, 0.279654779, 0.279762242, 0.277046962], "train_time_seconds": 71.97123666199998, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 233110.01419624328, "peak_memory_bytes": 6314994176, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "ademamix", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.999, "adema_alpha": 8.0, "adema_beta3_warmup": 4096, "adema_alpha_warmup": 4096, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216/run.log new file mode 100644 index 0000000..278bb7b --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216/run.log @@ -0,0 +1,63 @@ +Starting Experiment: sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adopt', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +ADOPT ( +Parameter Group 0 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.0001 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.0001 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.094 iter_dt=2.72e-01s lr=1.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.878 iter_dt=2.73e-01s lr=1.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.562 iter_dt=2.76e-01s lr=1.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.488 iter_dt=2.72e-01s lr=1.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.294 iter_dt=2.74e-01s lr=1.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.216 val_pp=500.891 val_acc=0.171034 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216/summary.json new file mode 100644 index 0000000..f4e72f4 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.094470977783203, 6.878191947937012, 6.562315940856934, 6.487613677978516, 6.293927192687988], "val_loss": [6.21639347076416], "val_pp": [500.8913890653622], "val_acc": [0.1710338592529297], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.21639347076416, "val_perplexity": 500.8913890653622, "val_accuracy": 0.1710338592529297, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.271540582, 0.272863276, 0.276279793, 0.272178337, 0.274339708], "train_time_seconds": 70.27018207499995, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 238752.98888643176, "peak_memory_bytes": 6175400960, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "adopt", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..559632c --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,63 @@ +Starting Experiment: sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adopt', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +ADOPT ( +Parameter Group 0 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.0003 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.0003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.364 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.555 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.231 iter_dt=2.70e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.158 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=5.976 iter_dt=2.71e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.876 val_pp=356.393 val_acc=0.179474 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..83cbf1a --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.3639936447143555, 6.555488109588623, 6.231332778930664, 6.157642364501953, 5.9755167961120605], "val_loss": [5.876038551330566], "val_pp": [356.3931939656842], "val_acc": [0.179473876953125], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.876038551330566, "val_perplexity": 356.3931939656842, "val_accuracy": 0.179473876953125, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.269848295, 0.270213586, 0.269941617, 0.270834475, 0.270762805], "train_time_seconds": 69.73402755099995, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 240588.65648811104, "peak_memory_bytes": 6175400960, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "adopt", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..8ec8dad --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,63 @@ +Starting Experiment: sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adopt', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +ADOPT ( +Parameter Group 0 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.001 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.001 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.524 iter_dt=2.75e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.760 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.420 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.352 iter_dt=2.73e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.198 iter_dt=2.73e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.104 val_pp=447.420 val_acc=0.158995 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..5c36b22 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.524109840393066, 6.759858131408691, 6.420350074768066, 6.351698398590088, 6.198278427124023], "val_loss": [6.103500843048096], "val_pp": [447.4195466420294], "val_acc": [0.1589946746826172], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.103500843048096, "val_perplexity": 447.4195466420294, "val_accuracy": 0.1589946746826172, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.275090097, 0.271822524, 0.271449355, 0.272765713, 0.272544072], "train_time_seconds": 69.85458485299993, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 240173.44080285513, "peak_memory_bytes": 6175400960, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "adopt", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..a4e94de --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,63 @@ +Starting Experiment: sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'adopt', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +ADOPT ( +Parameter Group 0 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.003 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + decouple: True + eps: 1e-06 + lr: 0.003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.546 iter_dt=2.72e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.836 iter_dt=2.73e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.568 iter_dt=2.73e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.583 iter_dt=2.75e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.451 iter_dt=2.79e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.370 val_pp=584.002 val_acc=0.136562 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..94afa25 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.545925140380859, 6.836182594299316, 6.568281173706055, 6.582737445831299, 6.450787544250488], "val_loss": [6.369908332824707], "val_pp": [584.0017901147743], "val_acc": [0.13656234741210938], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.369908332824707, "val_perplexity": 584.0017901147743, "val_accuracy": 0.13656234741210938, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.272228522, 0.27263125, 0.273167282, 0.275047938, 0.278597696], "train_time_seconds": 70.62556766500003, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 237551.59150832988, "peak_memory_bytes": 6175400960, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "adopt", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..61b3450 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,70 @@ +Starting Experiment: sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': 'nccl', 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'd-muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Distributed DataReader Initialized for Worker 0/1 +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +DistributedMuon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.0003 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.1 + +Parameter Group 1 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.0003 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.481 iter_dt=2.96e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.495 iter_dt=2.81e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.233 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.209 iter_dt=2.82e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.126 iter_dt=2.80e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.031 val_pp=415.935 val_acc=0.161667 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..1d2d66d --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.480681419372559, 6.49476432800293, 6.233405113220215, 6.208542823791504, 6.126290798187256], "val_loss": [6.030533790588379], "val_pp": [415.9353061617444], "val_acc": [0.1616668701171875], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.030533790588379, "val_perplexity": 415.9353061617444, "val_accuracy": 0.1616668701171875, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.29575757, 0.281179162, 0.283134459, 0.282185696, 0.28035137], "train_time_seconds": 73.275967925, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 228959.3228870337, "peak_memory_bytes": 6252898816, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": "nccl", "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "d-muon", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.95, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": true, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..4f97e2a --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,70 @@ +Starting Experiment: sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': 'nccl', 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'd-muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Distributed DataReader Initialized for Worker 0/1 +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +DistributedMuon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.001 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.1 + +Parameter Group 1 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.001 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.782 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.065 iter_dt=2.90e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=5.830 iter_dt=2.80e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.658 iter_dt=2.81e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.561 iter_dt=2.81e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.397 val_pp=220.810 val_acc=0.199068 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..1b1762c --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [6.78244686126709, 6.06502628326416, 5.8296051025390625, 5.657920837402344, 5.560568809509277], "val_loss": [5.397306442260742], "val_pp": [220.81004604229818], "val_acc": [0.1990680694580078], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.397306442260742, "val_perplexity": 220.81004604229818, "val_accuracy": 0.1990680694580078, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.285272137, 0.28983882, 0.279538589, 0.281257458, 0.280841382], "train_time_seconds": 72.49320627200002, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 231431.56252532976, "peak_memory_bytes": 6386240000, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": "nccl", "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "d-muon", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.95, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": true, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..0326313 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,70 @@ +Starting Experiment: sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': 'nccl', 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'd-muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Distributed DataReader Initialized for Worker 0/1 +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +DistributedMuon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.003 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.1 + +Parameter Group 1 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.003 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.577 iter_dt=2.83e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=5.880 iter_dt=2.84e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=5.535 iter_dt=2.80e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.348 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.313 iter_dt=2.83e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.128 val_pp=168.674 val_acc=0.215315 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..fd0c84d --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [6.576503276824951, 5.879572868347168, 5.5348052978515625, 5.348039627075195, 5.313028335571289], "val_loss": [5.1279730796813965], "val_pp": [168.6742989582807], "val_acc": [0.2153148651123047], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.1279730796813965, "val_perplexity": 168.6742989582807, "val_accuracy": 0.2153148651123047, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.283009269, 0.283978591, 0.27987217, 0.281737279, 0.282921861], "train_time_seconds": 72.539968561, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 231282.37208831674, "peak_memory_bytes": 6252898816, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": "nccl", "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "d-muon", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.95, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": true, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216/run.log new file mode 100644 index 0000000..8da4f22 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216/run.log @@ -0,0 +1,70 @@ +Starting Experiment: sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': 'nccl', 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.009, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'd-muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Distributed DataReader Initialized for Worker 0/1 +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +DistributedMuon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.009 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.1 + +Parameter Group 1 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + lr: 0.009 + matched_adamw_rms: 0.2 + momentum: 0.95 + nesterov: True + ns_steps: 5 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.713 iter_dt=2.79e-01s lr=9.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=5.791 iter_dt=2.79e-01s lr=9.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=5.484 iter_dt=2.79e-01s lr=9.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.383 iter_dt=2.80e-01s lr=9.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.356 iter_dt=2.81e-01s lr=9.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.192 val_pp=179.777 val_acc=0.210426 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216/summary.json new file mode 100644 index 0000000..18d5bf7 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [6.712637901306152, 5.791131973266602, 5.483846664428711, 5.38294792175293, 5.356256484985352], "val_loss": [5.191718101501465], "val_pp": [179.77653533712166], "val_acc": [0.21042633056640625], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.191718101501465, "val_perplexity": 179.77653533712166, "val_accuracy": 0.21042633056640625, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.278986461, 0.27923507, 0.27869423, 0.28000099, 0.28090079], "train_time_seconds": 72.07763624499998, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 232765.90179750565, "peak_memory_bytes": 6252898816, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": "nccl", "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.009, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "d-muon", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.95, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": true, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..4f113bd --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,61 @@ +Starting Experiment: sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lamb', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lamb ( +Parameter Group 0 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.0003 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.0003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=10.391 iter_dt=2.79e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=10.190 iter_dt=2.80e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=10.073 iter_dt=2.79e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=10.000 iter_dt=2.80e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=9.877 iter_dt=2.82e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=9.872 val_pp=19389.059 val_acc=0.088730 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..9db854f --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [10.390525817871094, 10.189567565917969, 10.072818756103516, 10.000383377075195, 9.876977920532227], "val_loss": [9.87247085571289], "val_pp": [19389.058962766347], "val_acc": [0.0887298583984375], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 9.87247085571289, "val_perplexity": 19389.058962766347, "val_accuracy": 0.0887298583984375, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.27916086, 0.27999209, 0.279309544, 0.27957942, 0.282096392], "train_time_seconds": 72.16825172200001, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 232473.63764093482, "peak_memory_bytes": 6175793152, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "lamb", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..f1c0b60 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,61 @@ +Starting Experiment: sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lamb', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lamb ( +Parameter Group 0 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.001 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.001 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=10.191 iter_dt=2.82e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=9.783 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=9.579 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=9.445 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=9.274 iter_dt=2.84e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=9.245 val_pp=10351.511 val_acc=0.131166 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..7c67aa8 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [10.191261291503906, 9.783426284790039, 9.579458236694336, 9.444581985473633, 9.274479866027832], "val_loss": [9.244894027709961], "val_pp": [10351.511318109584], "val_acc": [0.1311664581298828], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 9.244894027709961, "val_perplexity": 10351.511318109584, "val_accuracy": 0.1311664581298828, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.281687685, 0.282817926, 0.28360736, 0.283534056, 0.28372362], "train_time_seconds": 72.92484238200004, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 230061.73824986015, "peak_memory_bytes": 6175793152, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "lamb", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216/run.log new file mode 100644 index 0000000..657a941 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216/run.log @@ -0,0 +1,61 @@ +Starting Experiment: sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.027, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lamb', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lamb ( +Parameter Group 0 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.027 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.027 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.706 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=6.748 iter_dt=2.82e-01s lr=2.70e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=6.378 iter_dt=2.88e-01s lr=2.70e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=6.262 iter_dt=2.87e-01s lr=2.70e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=6.082 iter_dt=2.84e-01s lr=2.70e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.967 val_pp=390.174 val_acc=0.171759 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216/summary.json new file mode 100644 index 0000000..0586ae7 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.7056121826171875, 6.748311996459961, 6.378097057342529, 6.262062072753906, 6.081888198852539], "val_loss": [5.966597557067871], "val_pp": [390.17429268796366], "val_acc": [0.17175865173339844], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.966597557067871, "val_perplexity": 390.17429268796366, "val_accuracy": 0.17175865173339844, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.282422909, 0.282160309, 0.288270027, 0.287167892, 0.284478405], "train_time_seconds": 73.24245941699994, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 229064.07203614363, "peak_memory_bytes": 6175793152, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.027, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "lamb", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..4b91c67 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,61 @@ +Starting Experiment: sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lamb', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lamb ( +Parameter Group 0 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.003 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=9.845 iter_dt=2.91e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=9.185 iter_dt=2.88e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=8.729 iter_dt=2.90e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=8.418 iter_dt=2.91e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=8.025 iter_dt=2.94e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.944 val_pp=2819.426 val_acc=0.156244 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..980d75f --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [9.844831466674805, 9.184578895568848, 8.729259490966797, 8.418334007263184, 8.024789810180664], "val_loss": [7.944293975830078], "val_pp": [2819.4260985800843], "val_acc": [0.15624427795410156], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 7.944293975830078, "val_perplexity": 2819.4260985800843, "val_accuracy": 0.15624427795410156, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.291067021, 0.287528932, 0.290236592, 0.290626835, 0.294285094], "train_time_seconds": 75.17631810899998, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 223171.557506638, "peak_memory_bytes": 6175793152, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "lamb", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216/run.log new file mode 100644 index 0000000..1aeb4ea --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216/run.log @@ -0,0 +1,61 @@ +Starting Experiment: sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.081, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lamb', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lamb ( +Parameter Group 0 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.081 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.081 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.642 iter_dt=2.87e-01s lr=8.10e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=6.959 iter_dt=2.85e-01s lr=8.10e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=6.664 iter_dt=2.82e-01s lr=8.10e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=6.655 iter_dt=2.83e-01s lr=8.10e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=6.605 iter_dt=2.83e-01s lr=8.10e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.549 val_pp=698.800 val_acc=0.122236 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216/summary.json new file mode 100644 index 0000000..f69d065 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.641925811767578, 6.95856237411499, 6.6643571853637695, 6.655353546142578, 6.6048994064331055], "val_loss": [6.549368381500244], "val_pp": [698.7995791757087], "val_acc": [0.12223625183105469], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.549368381500244, "val_perplexity": 698.7995791757087, "val_accuracy": 0.12223625183105469, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.287091051, 0.284786487, 0.281699409, 0.282924334, 0.282775145], "train_time_seconds": 73.31413933900005, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 228840.1139434126, "peak_memory_bytes": 6175793152, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.081, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "lamb", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216/run.log new file mode 100644 index 0000000..bef6f90 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216/run.log @@ -0,0 +1,61 @@ +Starting Experiment: sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.009, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lamb', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lamb ( +Parameter Group 0 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.009 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + eps: 1e-06 + lr: 0.009 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=9.221 iter_dt=2.79e-01s lr=9.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.767 iter_dt=2.79e-01s lr=9.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.940 iter_dt=2.80e-01s lr=9.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.693 iter_dt=2.84e-01s lr=9.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.400 iter_dt=2.80e-01s lr=9.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.308 val_pp=548.823 val_acc=0.175278 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216/summary.json new file mode 100644 index 0000000..839a328 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [9.220962524414062, 7.7673869132995605, 6.9402875900268555, 6.692530155181885, 6.400154113769531], "val_loss": [6.3077802658081055], "val_pp": [548.8230206950632], "val_acc": [0.1752777099609375], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.3077802658081055, "val_perplexity": 548.8230206950632, "val_accuracy": 0.1752777099609375, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.279259364, 0.279489859, 0.280310843, 0.284185557, 0.280148626], "train_time_seconds": 72.42151405900002, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 231660.6635195726, "peak_memory_bytes": 6175793152, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.009, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "lamb", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216/run.log new file mode 100644 index 0000000..b1d5872 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216/run.log @@ -0,0 +1,59 @@ +Starting Experiment: sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.99, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lion ( +Parameter Group 0 + betas: (0.9, 0.99) + lr: 0.0001 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.99) + lr: 0.0001 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.618 iter_dt=2.74e-01s lr=1.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=7.342 iter_dt=2.83e-01s lr=1.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.871 iter_dt=2.73e-01s lr=1.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.733 iter_dt=2.75e-01s lr=1.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.512 iter_dt=2.74e-01s lr=1.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.423 val_pp=615.991 val_acc=0.147200 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216/summary.json new file mode 100644 index 0000000..bd6db35 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.617734909057617, 7.341650485992432, 6.871392250061035, 6.733397960662842, 6.511661529541016], "val_loss": [6.423236846923828], "val_pp": [615.9911067382695], "val_acc": [0.1471996307373047], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.423236846923828, "val_perplexity": 615.9911067382695, "val_accuracy": 0.1471996307373047, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.273605816, 0.282539472, 0.273374294, 0.275052707, 0.274477341], "train_time_seconds": 70.83748797199996, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 236840.9225159361, "peak_memory_bytes": 6039584256, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "lion", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.99, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..6b20e05 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,59 @@ +Starting Experiment: sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.99, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lion ( +Parameter Group 0 + betas: (0.9, 0.99) + lr: 0.0003 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.99) + lr: 0.0003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.614 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.893 iter_dt=2.76e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.630 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.554 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.389 iter_dt=2.74e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.277 val_pp=531.994 val_acc=0.146252 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..2ba7277 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.6139421463012695, 6.892988204956055, 6.629602909088135, 6.554364204406738, 6.388577938079834], "val_loss": [6.276636123657227], "val_pp": [531.9938353895492], "val_acc": [0.14625167846679688], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.276636123657227, "val_perplexity": 531.9938353895492, "val_accuracy": 0.14625167846679688, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.272992573, 0.275510045, 0.275121636, 0.27390767, 0.274267264], "train_time_seconds": 70.65828603599996, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 237441.59307023257, "peak_memory_bytes": 6039584256, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "lion", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.99, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..ca1781d --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,59 @@ +Starting Experiment: sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.99, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lion ( +Parameter Group 0 + betas: (0.9, 0.99) + lr: 0.001 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.99) + lr: 0.001 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.579 iter_dt=2.95e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.019 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.878 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.972 iter_dt=2.74e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.916 iter_dt=2.76e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.849 val_pp=942.977 val_acc=0.099554 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..5e00382 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.57949161529541, 7.019062042236328, 6.877532005310059, 6.972165107727051, 6.915810585021973], "val_loss": [6.849046230316162], "val_pp": [942.9767465226221], "val_acc": [0.09955406188964844], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.849046230316162, "val_perplexity": 942.9767465226221, "val_accuracy": 0.09955406188964844, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.295448793, 0.27435868, 0.27414744, 0.273967471, 0.276065448], "train_time_seconds": 71.19586444, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 235648.74353255343, "peak_memory_bytes": 6039584256, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "lion", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.99, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..a4a3347 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,59 @@ +Starting Experiment: sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'lion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.99, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Lion ( +Parameter Group 0 + betas: (0.9, 0.99) + lr: 0.003 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.99) + lr: 0.003 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.017 iter_dt=2.71e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.599 iter_dt=2.73e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=7.506 iter_dt=2.73e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=7.631 iter_dt=2.71e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=7.413 iter_dt=2.72e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.386 val_pp=1613.586 val_acc=0.071882 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..42ee677 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.017406463623047, 7.598730087280273, 7.50596809387207, 7.630977630615234, 7.41286039352417], "val_loss": [7.386219024658203], "val_pp": [1613.585590064328], "val_acc": [0.07188224792480469], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 7.386219024658203, "val_perplexity": 1613.585590064328, "val_accuracy": 0.07188224792480469, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.271415485, 0.272953884, 0.273279452, 0.270895599, 0.271547349], "train_time_seconds": 69.997784646, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 239682.09972426217, "peak_memory_bytes": 6039584256, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "lion", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.99, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216/run.log new file mode 100644 index 0000000..d02d4ba --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216/run.log @@ -0,0 +1,71 @@ +Starting Experiment: sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'mars', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.0003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +MARS ( +Parameter Group 0 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.0003 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.1 + weight_decay_1d: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.0003 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.0 + weight_decay_1d: 0.1 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.608 iter_dt=2.78e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.717 iter_dt=2.78e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.388 iter_dt=2.78e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.349 iter_dt=2.78e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.189 iter_dt=2.78e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.105 val_pp=448.299 val_acc=0.176981 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216/summary.json new file mode 100644 index 0000000..4677e69 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.608182907104492, 6.716814041137695, 6.387589454650879, 6.348698139190674, 6.188875198364258], "val_loss": [6.105465412139893], "val_pp": [448.2993966426686], "val_acc": [0.17698097229003906], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.105465412139893, "val_perplexity": 448.2993966426686, "val_accuracy": 0.17698097229003906, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.278094101, 0.278218036, 0.278050351, 0.278462188, 0.278432434], "train_time_seconds": 71.78387870099995, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 233718.43795013955, "peak_memory_bytes": 6324693504, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "mars", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.0003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..b1c89da --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,71 @@ +Starting Experiment: sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'mars', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.0009, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +MARS ( +Parameter Group 0 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.0009 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.1 + weight_decay_1d: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.0009 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.0 + weight_decay_1d: 0.1 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.226 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.450 iter_dt=2.81e-01s lr=9.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.169 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.108 iter_dt=2.82e-01s lr=9.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=5.946 iter_dt=2.82e-01s lr=9.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.844 val_pp=345.245 val_acc=0.178284 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..1b9206e --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.226407051086426, 6.449817657470703, 6.169248580932617, 6.107900619506836, 5.946225166320801], "val_loss": [5.844259262084961], "val_pp": [345.24535249628593], "val_acc": [0.17828369140625], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.844259262084961, "val_perplexity": 345.24535249628593, "val_accuracy": 0.17828369140625, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.280726773, 0.281159247, 0.28187457, 0.281518269, 0.281785598], "train_time_seconds": 72.59430473599997, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 231109.25934221494, "peak_memory_bytes": 6324693504, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "mars", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.0009, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..e277bde --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,71 @@ +Starting Experiment: sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'mars', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +MARS ( +Parameter Group 0 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.003 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.1 + weight_decay_1d: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.003 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.0 + weight_decay_1d: 0.1 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.338 iter_dt=2.78e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.713 iter_dt=2.77e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.482 iter_dt=2.77e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.435 iter_dt=2.78e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.293 iter_dt=2.77e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.187 val_pp=486.185 val_acc=0.151686 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..7cb0561 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.338011741638184, 6.712757587432861, 6.4822187423706055, 6.434700012207031, 6.293219566345215], "val_loss": [6.1865925788879395], "val_pp": [486.18461472526104], "val_acc": [0.1516857147216797], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.1865925788879395, "val_perplexity": 486.18461472526104, "val_accuracy": 0.1516857147216797, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.277637331, 0.277358871, 0.277009068, 0.277743893, 0.276763088], "train_time_seconds": 71.45638488799999, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 234789.599645944, "peak_memory_bytes": 6324693504, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "mars", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..3c19bc7 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,71 @@ +Starting Experiment: sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'mars', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.009, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +MARS ( +Parameter Group 0 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.009 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.1 + weight_decay_1d: 0.1 + +Parameter Group 1 + amsgrad: False + betas: (0.95, 0.99) + eps: 1e-08 + gamma: 0.025 + lr: 0.009 + mars_type: mars-adamw + optimize_1d: False + weight_decay: 0.0 + weight_decay_1d: 0.1 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.712 iter_dt=2.78e-01s lr=9.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.038 iter_dt=2.78e-01s lr=9.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.808 iter_dt=2.79e-01s lr=9.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.794 iter_dt=2.78e-01s lr=9.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.670 iter_dt=2.79e-01s lr=9.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.590 val_pp=727.953 val_acc=0.120892 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..3531651 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.712274551391602, 7.037928581237793, 6.807950019836426, 6.7942094802856445, 6.670461654663086], "val_loss": [6.590240478515625], "val_pp": [727.9526796275507], "val_acc": [0.12089157104492188], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.590240478515625, "val_perplexity": 727.9526796275507, "val_accuracy": 0.12089157104492188, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.277613512, 0.278103558, 0.278541482, 0.277919388, 0.27865329], "train_time_seconds": 71.51491461100001, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 234597.44154430446, "peak_memory_bytes": 6324693504, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "mars", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.009, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..0df562c --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,49 @@ +Starting Experiment: sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.006, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon-pytorch', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.0003, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: + +Train: Iter=50 (0.006104 effective passes) train_loss=7.390 iter_dt=2.77e-01s lr=6.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.445 iter_dt=2.77e-01s lr=6.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.154 iter_dt=2.77e-01s lr=6.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.076 iter_dt=2.77e-01s lr=6.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.913 iter_dt=2.78e-01s lr=6.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.794 val_pp=328.169 val_acc=0.192841 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..6f2da85 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.389583587646484, 6.444538116455078, 6.153596878051758, 6.076385021209717, 5.912977695465088], "val_loss": [5.793531894683838], "val_pp": [328.1687630800648], "val_acc": [0.192840576171875], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.793531894683838, "val_perplexity": 328.1687630800648, "val_accuracy": 0.192840576171875, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.277031636, 0.277215685, 0.277199882, 0.277190399, 0.277864274], "train_time_seconds": 71.583995558, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 234371.04717641082, "peak_memory_bytes": 6117270528, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.006, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "muon-pytorch", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.95, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": true, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.0003, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..1f5faa4 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,49 @@ +Starting Experiment: sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.02, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon-pytorch', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: + +Train: Iter=50 (0.006104 effective passes) train_loss=6.818 iter_dt=2.78e-01s lr=2.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=6.057 iter_dt=2.78e-01s lr=2.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=5.715 iter_dt=2.77e-01s lr=2.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=5.620 iter_dt=2.77e-01s lr=2.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=5.505 iter_dt=2.78e-01s lr=2.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.356 val_pp=211.939 val_acc=0.205448 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..0f05b8b --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [6.817802429199219, 6.056707382202148, 5.71474027633667, 5.620233535766602, 5.5051984786987305], "val_loss": [5.356301784515381], "val_pp": [211.9389330623356], "val_acc": [0.20544815063476562], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.356301784515381, "val_perplexity": 211.9389330623356, "val_accuracy": 0.20544815063476562, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.277730077, 0.277780163, 0.277065708, 0.277395697, 0.27772221], "train_time_seconds": 71.78204306099995, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 233724.41469439398, "peak_memory_bytes": 6117270528, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.02, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "muon-pytorch", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.95, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": true, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..69d21e2 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,49 @@ +Starting Experiment: sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.06, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon-pytorch', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.003, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: + +Train: Iter=50 (0.006104 effective passes) train_loss=6.799 iter_dt=2.77e-01s lr=6.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=5.850 iter_dt=2.77e-01s lr=6.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=5.538 iter_dt=2.76e-01s lr=6.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=5.469 iter_dt=2.76e-01s lr=6.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=5.429 iter_dt=2.77e-01s lr=6.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.283 val_pp=197.032 val_acc=0.205511 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..a6a262c --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [6.798731803894043, 5.849853038787842, 5.538264274597168, 5.469029426574707, 5.429256439208984], "val_loss": [5.283370018005371], "val_pp": [197.03206148135158], "val_acc": [0.20551109313964844], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.283370018005371, "val_perplexity": 197.03206148135158, "val_accuracy": 0.20551109313964844, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.27674231, 0.276523505, 0.276180374, 0.276060213, 0.276700634], "train_time_seconds": 71.56041207699997, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 234448.2866021997, "peak_memory_bytes": 6117270528, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.06, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "muon-pytorch", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.95, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": true, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.003, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216/run.log new file mode 100644 index 0000000..3be81a0 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216/run.log @@ -0,0 +1,49 @@ +Starting Experiment: sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.18, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon-pytorch', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.009, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: + +Train: Iter=50 (0.006104 effective passes) train_loss=6.990 iter_dt=2.76e-01s lr=1.80e-01 +Train: Iter=100 (0.012207 effective passes) train_loss=6.290 iter_dt=2.77e-01s lr=1.80e-01 +Train: Iter=150 (0.018311 effective passes) train_loss=6.096 iter_dt=2.76e-01s lr=1.80e-01 +Train: Iter=200 (0.024414 effective passes) train_loss=6.027 iter_dt=2.77e-01s lr=1.80e-01 +Train: Iter=250 (0.030518 effective passes) train_loss=6.029 iter_dt=2.77e-01s lr=1.80e-01 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.898 val_pp=364.330 val_acc=0.166126 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216/summary.json new file mode 100644 index 0000000..3016582 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [6.990287780761719, 6.290014743804932, 6.09600830078125, 6.026956558227539, 6.029223918914795], "val_loss": [5.898062705993652], "val_pp": [364.32952211229554], "val_acc": [0.16612625122070312], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.898062705993652, "val_perplexity": 364.32952211229554, "val_accuracy": 0.16612625122070312, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.276392296, 0.276762662, 0.276430682, 0.276567118, 0.277456974], "train_time_seconds": 71.63904080699997, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 234190.96362832192, "peak_memory_bytes": 6117270528, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.18, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "muon-pytorch", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.95, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": true, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.009, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..184afe0 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,60 @@ +Starting Experiment: sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 0.003, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Muon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + adamw_lr: 0.0003 + adamw_lr_ratio: 0.09999999999999999 + adamw_wd: 0.1 + lr: 0.003 + momentum: 0.95 + nesterov: True + ns_steps: 5 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.384 iter_dt=2.91e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.422 iter_dt=2.92e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.163 iter_dt=2.92e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.094 iter_dt=2.96e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.967 iter_dt=2.93e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.848 val_pp=346.491 val_acc=0.183538 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..dea6e90 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.384439945220947, 6.421547889709473, 6.163295745849609, 6.0937700271606445, 5.9673380851745605], "val_loss": [5.847861289978027], "val_pp": [346.4911774490298], "val_acc": [0.18353843688964844], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.847861289978027, "val_perplexity": 346.4911774490298, "val_accuracy": 0.18353843688964844, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.290976824, 0.292441834, 0.29151777, 0.296114758, 0.292962155], "train_time_seconds": 75.548639173, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 222071.7167596043, "peak_memory_bytes": 6117368832, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "muon", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.95, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": true, "muon_ns_steps": 5, "muon_lr_factor": 0.003, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..49586bc --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,60 @@ +Starting Experiment: sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 0.01, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Muon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + adamw_lr: 0.001 + adamw_lr_ratio: 0.1 + adamw_wd: 0.1 + lr: 0.01 + momentum: 0.95 + nesterov: True + ns_steps: 5 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.762 iter_dt=2.89e-01s lr=1.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=6.032 iter_dt=2.88e-01s lr=1.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=5.684 iter_dt=2.86e-01s lr=1.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=5.522 iter_dt=2.87e-01s lr=1.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=5.414 iter_dt=2.88e-01s lr=1.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.261 val_pp=192.755 val_acc=0.210743 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..204d8af --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [6.762043476104736, 6.031713008880615, 5.683930397033691, 5.52216911315918, 5.413718223571777], "val_loss": [5.261423110961914], "val_pp": [192.75492663883713], "val_acc": [0.21074295043945312], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.261423110961914, "val_perplexity": 192.75492663883713, "val_accuracy": 0.21074295043945312, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.289248498, 0.288437783, 0.285918225, 0.286626008, 0.288296012], "train_time_seconds": 74.21483872099998, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 226062.82367696753, "peak_memory_bytes": 6117368832, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "muon", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.95, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": true, "muon_ns_steps": 5, "muon_lr_factor": 0.01, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..9f20ace --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,60 @@ +Starting Experiment: sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 0.03, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Muon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + adamw_lr: 0.003 + adamw_lr_ratio: 0.1 + adamw_wd: 0.1 + lr: 0.03 + momentum: 0.95 + nesterov: True + ns_steps: 5 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.665 iter_dt=2.87e-01s lr=3.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=5.831 iter_dt=2.86e-01s lr=3.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=5.494 iter_dt=2.95e-01s lr=3.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=5.375 iter_dt=2.88e-01s lr=3.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=5.330 iter_dt=2.87e-01s lr=3.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.157 val_pp=173.725 val_acc=0.213518 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..d06c013 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [6.664637565612793, 5.830648422241211, 5.4941253662109375, 5.3751959800720215, 5.329990386962891], "val_loss": [5.157479286193848], "val_pp": [173.72538702515928], "val_acc": [0.2135181427001953], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.157479286193848, "val_perplexity": 173.72538702515928, "val_accuracy": 0.2135181427001953, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.287391317, 0.285897121, 0.295173976, 0.288163515, 0.286507039], "train_time_seconds": 74.58441240600004, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 224942.65837576453, "peak_memory_bytes": 6117368832, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "muon", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.95, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": true, "muon_ns_steps": 5, "muon_lr_factor": 0.03, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216/run.log new file mode 100644 index 0000000..1d5b13a --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216/run.log @@ -0,0 +1,60 @@ +Starting Experiment: sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.009, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'muon', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.8, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.95, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': True, 'muon_ns_steps': 5, 'muon_lr_factor': 0.09, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Muon ( +Parameter Group 0 + adamw_betas: (0.8, 0.999) + adamw_eps: 1e-08 + adamw_lr: 0.009 + adamw_lr_ratio: 0.09999999999999999 + adamw_wd: 0.1 + lr: 0.09 + momentum: 0.95 + nesterov: True + ns_steps: 5 +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.844 iter_dt=2.88e-01s lr=9.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=5.939 iter_dt=2.85e-01s lr=9.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=5.651 iter_dt=2.84e-01s lr=9.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=5.608 iter_dt=2.86e-01s lr=9.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=5.650 iter_dt=2.86e-01s lr=9.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.522 val_pp=250.195 val_acc=0.188259 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216/summary.json new file mode 100644 index 0000000..2549d67 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [6.844460487365723, 5.939065456390381, 5.650857925415039, 5.60845947265625, 5.649868965148926], "val_loss": [5.522243499755859], "val_pp": [250.19479268436115], "val_acc": [0.18825912475585938], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.522243499755859, "val_perplexity": 250.19479268436115, "val_accuracy": 0.18825912475585938, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.288489468, 0.2848689, 0.284243287, 0.285558057, 0.285772374], "train_time_seconds": 73.899425389, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 227027.6921868638, "peak_memory_bytes": 6117368832, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.009, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "muon", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.8, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.95, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": true, "muon_ns_steps": 5, "muon_lr_factor": 0.09, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216/run.log new file mode 100644 index 0000000..163b7f1 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216/run.log @@ -0,0 +1,91 @@ +Starting Experiment: sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.1, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'prodigy', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': True, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Using decoupled weight decay + +Optimizer: +Prodigy ( +Parameter Group 0 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 0.1 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.1 + +Parameter Group 1 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 0.1 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=10.893 iter_dt=2.82e-01s lr=1.00e-01 +effective_lr=7.08e-08 +Train: Iter=100 (0.012207 effective passes) train_loss=10.163 iter_dt=2.82e-01s lr=1.00e-01 +effective_lr=3.53e-06 +Train: Iter=150 (0.018311 effective passes) train_loss=7.152 iter_dt=2.82e-01s lr=1.00e-01 +effective_lr=8.62e-05 +Train: Iter=200 (0.024414 effective passes) train_loss=6.522 iter_dt=2.80e-01s lr=1.00e-01 +effective_lr=1.19e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.169 iter_dt=2.82e-01s lr=1.00e-01 +effective_lr=1.54e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.070 val_pp=432.473 val_acc=0.171778 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216/summary.json new file mode 100644 index 0000000..0187597 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [10.893074035644531, 10.163013458251953, 7.152050018310547, 6.522450923919678, 6.168804168701172], "val_loss": [6.069522857666016], "val_pp": [432.47251489850487], "val_acc": [0.17177772521972656], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.069522857666016, "val_perplexity": 432.47251489850487, "val_accuracy": 0.17177772521972656, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.281916949, 0.281865184, 0.282200257, 0.280000314, 0.281659404], "train_time_seconds": 72.63071371999999, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 230993.4067931393, "peak_memory_bytes": 6451916800, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.1, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "prodigy", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": true, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..8fa7afc --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,91 @@ +Starting Experiment: sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.3, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'prodigy', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': True, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Using decoupled weight decay + +Optimizer: +Prodigy ( +Parameter Group 0 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 0.3 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.1 + +Parameter Group 1 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 0.3 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=10.238 iter_dt=2.84e-01s lr=3.00e-01 +effective_lr=6.14e-06 +Train: Iter=100 (0.012207 effective passes) train_loss=7.053 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=1.58e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.421 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=1.90e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.258 iter_dt=2.81e-01s lr=3.00e-01 +effective_lr=2.45e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.011 iter_dt=2.80e-01s lr=3.00e-01 +effective_lr=3.60e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.900 val_pp=364.954 val_acc=0.172651 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..70c6ac2 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [10.23794174194336, 7.053047180175781, 6.42092227935791, 6.2583489418029785, 6.011272430419922], "val_loss": [5.899776458740234], "val_pp": [364.95442772504396], "val_acc": [0.1726512908935547], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.899776458740234, "val_perplexity": 364.95442772504396, "val_accuracy": 0.1726512908935547, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.283965852, 0.279852419, 0.280373023, 0.2808116, 0.280224128], "train_time_seconds": 72.33771752700002, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 231929.0208975409, "peak_memory_bytes": 6451916800, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.3, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "prodigy", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": true, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..2a39ab8 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,91 @@ +Starting Experiment: sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 1.0, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'prodigy', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': True, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Using decoupled weight decay + +Optimizer: +Prodigy ( +Parameter Group 0 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 1.0 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.1 + +Parameter Group 1 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 1.0 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.590 iter_dt=2.82e-01s lr=1.00e+00 +effective_lr=2.61e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=7.034 iter_dt=2.84e-01s lr=1.00e+00 +effective_lr=3.61e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.678 iter_dt=2.82e-01s lr=1.00e+00 +effective_lr=4.36e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.577 iter_dt=2.83e-01s lr=1.00e+00 +effective_lr=4.96e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.417 iter_dt=2.82e-01s lr=1.00e+00 +effective_lr=5.48e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.313 val_pp=551.823 val_acc=0.142332 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..d09129a --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.590337753295898, 7.033950328826904, 6.6776251792907715, 6.577483654022217, 6.4171905517578125], "val_loss": [6.313230991363525], "val_pp": [551.822670047611], "val_acc": [0.1423320770263672], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.313230991363525, "val_perplexity": 551.822670047611, "val_accuracy": 0.1423320770263672, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.281740826, 0.283620463, 0.281929926, 0.282552598, 0.282377933], "train_time_seconds": 72.822075675, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 230386.4019871609, "peak_memory_bytes": 6451916800, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 1.0, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "prodigy", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": true, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..329fb87 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,91 @@ +Starting Experiment: sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 3.0, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'prodigy', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': True, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Using decoupled weight decay + +Optimizer: +Prodigy ( +Parameter Group 0 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 3.0 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.1 + +Parameter Group 1 + beta3: None + betas: (0.9, 0.999) + d: 1e-06 + d0: 1e-06 + d_coef: 1.0 + d_max: 1e-06 + d_numerator: 0.0 + decouple: True + eps: 1e-08 + fsdp_in_use: False + growth_rate: inf + k: 0 + lr: 3.0 + safeguard_warmup: False + use_bias_correction: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.813 iter_dt=2.79e-01s lr=3.00e+00 +effective_lr=5.69e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=7.152 iter_dt=2.77e-01s lr=3.00e+00 +effective_lr=7.87e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.765 iter_dt=2.77e-01s lr=3.00e+00 +effective_lr=9.50e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.645 iter_dt=2.77e-01s lr=3.00e+00 +effective_lr=1.08e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.469 iter_dt=2.77e-01s lr=3.00e+00 +effective_lr=1.20e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.387 val_pp=593.959 val_acc=0.135414 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..175f7ae --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.812776565551758, 7.152127265930176, 6.7648468017578125, 6.644747734069824, 6.469451904296875], "val_loss": [6.386815071105957], "val_pp": [593.9592860424405], "val_acc": [0.13541412353515625], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.386815071105957, "val_perplexity": 593.9592860424405, "val_accuracy": 0.13541412353515625, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.278853423, 0.277322115, 0.276749207, 0.276813908, 0.276999879], "train_time_seconds": 71.42516241400007, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 234892.23451470223, "peak_memory_bytes": 6451916800, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 3.0, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "prodigy", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": true, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216/run.log new file mode 100644 index 0000000..112e2ae --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216/run.log @@ -0,0 +1,66 @@ +Starting Experiment: sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion-light', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +ScionLight ( +Parameter Group 0 + lr: 0.0001 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.0001 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.513 iter_dt=2.80e-01s lr=1.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.831 iter_dt=2.89e-01s lr=1.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.682 iter_dt=2.92e-01s lr=1.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.680 iter_dt=2.88e-01s lr=1.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.649 iter_dt=2.88e-01s lr=1.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.559 val_pp=705.505 val_acc=0.138273 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216/summary.json new file mode 100644 index 0000000..a23aa19 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.513493537902832, 6.8313679695129395, 6.682455062866211, 6.679723739624023, 6.6492438316345215], "val_loss": [6.558917999267578], "val_pp": [705.5048087754254], "val_acc": [0.1382732391357422], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.558917999267578, "val_perplexity": 705.5048087754254, "val_accuracy": 0.1382732391357422, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.280438894, 0.288687573, 0.291764632, 0.287743599, 0.288134024], "train_time_seconds": 73.74715544899999, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 227496.44915595857, "peak_memory_bytes": 5909968384, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "scion-light", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..df44af5 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,66 @@ +Starting Experiment: sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion-light', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +ScionLight ( +Parameter Group 0 + lr: 0.0003 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.0003 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.336 iter_dt=2.93e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.691 iter_dt=2.79e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.536 iter_dt=2.79e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.580 iter_dt=2.79e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.600 iter_dt=2.80e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.454 val_pp=635.227 val_acc=0.142559 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..3717207 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.3360490798950195, 6.690928936004639, 6.5362138748168945, 6.57993221282959, 6.600055694580078], "val_loss": [6.453986167907715], "val_pp": [635.2266260353618], "val_acc": [0.14255905151367188], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.453986167907715, "val_perplexity": 635.2266260353618, "val_accuracy": 0.14255905151367188, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.29282326, 0.279442733, 0.27912991, 0.279340477, 0.279715144], "train_time_seconds": 72.206245666, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 232351.31317594516, "peak_memory_bytes": 5909968384, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "scion-light", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..34a0b58 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,66 @@ +Starting Experiment: sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion-light', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +ScionLight ( +Parameter Group 0 + lr: 0.001 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.001 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.197 iter_dt=2.84e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.647 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.505 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.542 iter_dt=2.83e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.537 iter_dt=2.84e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.479 val_pp=651.508 val_acc=0.115211 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..a648abe --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.196730613708496, 6.646890640258789, 6.504509449005127, 6.541581153869629, 6.537202835083008], "val_loss": [6.479294776916504], "val_pp": [651.5084839119285], "val_acc": [0.11521148681640625], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.479294776916504, "val_perplexity": 651.5084839119285, "val_accuracy": 0.11521148681640625, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.284189992, 0.283481679, 0.283381381, 0.2832817, 0.283972384], "train_time_seconds": 73.16113002599994, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 229318.71055077645, "peak_memory_bytes": 5909968384, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "scion-light", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..5668b1c --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,66 @@ +Starting Experiment: sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion-light', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +ScionLight ( +Parameter Group 0 + lr: 0.003 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.003 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.870 iter_dt=2.84e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.960 iter_dt=2.84e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.727 iter_dt=2.85e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.858 iter_dt=2.86e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.754 iter_dt=2.87e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.810 val_pp=907.229 val_acc=0.103092 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..c44439b --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.869873046875, 6.960330963134766, 6.7272233963012695, 6.8583784103393555, 6.75434684753418], "val_loss": [6.810400009155273], "val_pp": [907.2294800769788], "val_acc": [0.10309219360351562], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.810400009155273, "val_perplexity": 907.2294800769788, "val_accuracy": 0.10309219360351562, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.284435293, 0.28430184, 0.285396161, 0.285714216, 0.286843777], "train_time_seconds": 73.39754613599999, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 228580.06681739897, "peak_memory_bytes": 5909968384, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "scion-light", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216/run.log new file mode 100644 index 0000000..fc2b761 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216/run.log @@ -0,0 +1,66 @@ +Starting Experiment: sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +Scion ( +Parameter Group 0 + lr: 0.0001 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.0001 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.422 iter_dt=2.81e-01s lr=1.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.636 iter_dt=2.82e-01s lr=1.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.472 iter_dt=2.85e-01s lr=1.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.467 iter_dt=2.84e-01s lr=1.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.445 iter_dt=2.84e-01s lr=1.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.358 val_pp=577.189 val_acc=0.143938 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216/summary.json new file mode 100644 index 0000000..2be35df --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.422316551208496, 6.6358723640441895, 6.471502304077148, 6.466629505157471, 6.444538116455078], "val_loss": [6.358173370361328], "val_pp": [577.1886100048764], "val_acc": [0.1439380645751953], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.358173370361328, "val_perplexity": 577.1886100048764, "val_accuracy": 0.1439380645751953, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.280801032, 0.281521564, 0.285315937, 0.284337003, 0.284168927], "train_time_seconds": 73.12712026200005, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 229425.36147862166, "peak_memory_bytes": 6039584256, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "scion", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..0ca47f7 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,66 @@ +Starting Experiment: sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +Scion ( +Parameter Group 0 + lr: 0.0003 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.0003 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.070 iter_dt=2.81e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.437 iter_dt=2.81e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.296 iter_dt=2.81e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.345 iter_dt=2.82e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.367 iter_dt=2.83e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.255 val_pp=520.485 val_acc=0.140451 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..16620f3 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.070192337036133, 6.436588287353516, 6.295558929443359, 6.345375061035156, 6.36683988571167], "val_loss": [6.25476598739624], "val_pp": [520.4853700206932], "val_acc": [0.14045143127441406], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.25476598739624, "val_perplexity": 520.4853700206932, "val_accuracy": 0.14045143127441406, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.280921177, 0.281000252, 0.28056, 0.281827975, 0.283328191], "train_time_seconds": 72.59823264799998, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 231096.7552246907, "peak_memory_bytes": 6039584256, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "scion", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..5be09cf --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,66 @@ +Starting Experiment: sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +Scion ( +Parameter Group 0 + lr: 0.001 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.001 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=6.988 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.473 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.374 iter_dt=2.85e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.358 iter_dt=2.86e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.412 iter_dt=2.86e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.320 val_pp=555.530 val_acc=0.121758 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..1262d39 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [6.987722396850586, 6.472510814666748, 6.373837471008301, 6.357669830322266, 6.411905288696289], "val_loss": [6.319926738739014], "val_pp": [555.52993033717], "val_acc": [0.12175750732421875], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.319926738739014, "val_perplexity": 555.52993033717, "val_accuracy": 0.12175750732421875, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.284618779, 0.284774496, 0.28546788, 0.28623271, 0.286114059], "train_time_seconds": 73.90401923500002, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 227013.58023102646, "peak_memory_bytes": 6039584256, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "scion", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..a22a0f2 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,66 @@ +Starting Experiment: sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'scion', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M +Optimized parameters: 33479040 + +Optimizer: +Scion ( +Parameter Group 0 + lr: 0.003 + momentum: 0.9 + norm: Sign + norm_kwargs: {'normalized': False} + scale: 10.0 + unconstrained: False + +Parameter Group 1 + lr: 0.003 + momentum: 0.9 + norm: Auto + norm_kwargs: {'max': True} + scale: 3.0 + unconstrained: False +) +Train: Iter=50 (0.006104 effective passes) train_loss=9.169 iter_dt=2.85e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.972 iter_dt=2.85e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.762 iter_dt=2.86e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.747 iter_dt=2.84e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.572 iter_dt=2.85e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.582 val_pp=721.782 val_acc=0.113983 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..63fc61c --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [9.168752670288086, 6.971536636352539, 6.761782169342041, 6.746805667877197, 6.572116374969482], "val_loss": [6.581727027893066], "val_pp": [721.7816004465782], "val_acc": [0.113983154296875], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.581727027893066, "val_perplexity": 721.7816004465782, "val_accuracy": 0.113983154296875, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.284517004, 0.284993607, 0.285609556, 0.283918991, 0.284534733], "train_time_seconds": 73.70046004400001, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 227640.58718200418, "peak_memory_bytes": 6039584256, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "scion", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..fcb4fce --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,79 @@ +Starting Experiment: sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.9999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdamWScheduleFree ( +Parameter Group 0 + betas: (0.9, 0.9999) + eps: 1e-08 + foreach: True + k: 0 + lr: 0.0003 + lr_max: -1.0 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + betas: (0.9, 0.9999) + eps: 1e-08 + foreach: True + k: 0 + lr: 0.0003 + lr_max: -1.0 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.649 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.686 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.388 iter_dt=2.71e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.382 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.255 iter_dt=2.75e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.212 val_pp=498.683 val_acc=0.170010 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..738f229 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.649140357971191, 6.685513496398926, 6.388017654418945, 6.381831169128418, 6.254584789276123], "val_loss": [6.211974143981934], "val_pp": [498.6826719365267], "val_acc": [0.17000961303710938], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.211974143981934, "val_perplexity": 498.6826719365267, "val_accuracy": 0.17000961303710938, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.270946775, 0.27077873, 0.271023088, 0.274179683, 0.274958055], "train_time_seconds": 69.91316692800001, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 239972.19318183648, "peak_memory_bytes": 6175400960, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "none", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sf-adamw", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.9999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..e8b4ccd --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,79 @@ +Starting Experiment: sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.9999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdamWScheduleFree ( +Parameter Group 0 + betas: (0.9, 0.9999) + eps: 1e-08 + foreach: True + k: 0 + lr: 0.001 + lr_max: -1.0 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + betas: (0.9, 0.9999) + eps: 1e-08 + foreach: True + k: 0 + lr: 0.001 + lr_max: -1.0 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.192 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.432 iter_dt=2.70e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.210 iter_dt=2.71e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.219 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.101 iter_dt=2.70e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.036 val_pp=418.182 val_acc=0.167719 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..ae85f19 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.191596031188965, 6.432206153869629, 6.210190773010254, 6.219433307647705, 6.101407051086426], "val_loss": [6.035920143127441], "val_pp": [418.1817234095075], "val_acc": [0.16771888732910156], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.035920143127441, "val_perplexity": 418.1817234095075, "val_accuracy": 0.16771888732910156, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.268765531, 0.270335071, 0.271398725, 0.269252133, 0.269794795], "train_time_seconds": 69.51470453899998, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 241347.7279557082, "peak_memory_bytes": 6175400960, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "none", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sf-adamw", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.9999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..74c0e82 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,79 @@ +Starting Experiment: sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-adamw', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.9999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +AdamWScheduleFree ( +Parameter Group 0 + betas: (0.9, 0.9999) + eps: 1e-08 + foreach: True + k: 0 + lr: 0.003 + lr_max: -1.0 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + betas: (0.9, 0.9999) + eps: 1e-08 + foreach: True + k: 0 + lr: 0.003 + lr_max: -1.0 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.357 iter_dt=2.71e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.801 iter_dt=2.79e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.580 iter_dt=2.72e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.577 iter_dt=2.72e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.477 iter_dt=2.72e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.418 val_pp=612.911 val_acc=0.132021 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..a937a42 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.356538772583008, 6.800962924957275, 6.579828262329102, 6.576943874359131, 6.477177619934082], "val_loss": [6.4182233810424805], "val_pp": [612.9105869064393], "val_acc": [0.1320209503173828], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.4182233810424805, "val_perplexity": 612.9105869064393, "val_accuracy": 0.1320209503173828, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.27065228, 0.27850033, 0.271843172, 0.272103317, 0.272192121], "train_time_seconds": 69.997612869, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 239682.6879139212, "peak_memory_bytes": 6175400960, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "none", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sf-adamw", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.9999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..fa8357f --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,77 @@ +Starting Experiment: sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGDScheduleFree ( +Parameter Group 0 + foreach: True + k: 0 + lr: 0.003 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + foreach: True + k: 0 + lr: 0.003 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=10.781 iter_dt=2.70e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=10.585 iter_dt=2.70e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=10.452 iter_dt=2.71e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=10.401 iter_dt=2.71e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=10.309 iter_dt=2.73e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=10.325 val_pp=30494.943 val_acc=0.034683 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..8e5392a --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [10.78094482421875, 10.585418701171875, 10.452232360839844, 10.40090560913086, 10.308727264404297], "val_loss": [10.325323104858398], "val_pp": [30494.943424849957], "val_acc": [0.0346832275390625], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 10.325323104858398, "val_perplexity": 30494.943424849957, "val_accuracy": 0.0346832275390625, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.270187507, 0.270266842, 0.270581038, 0.270715185, 0.272919413], "train_time_seconds": 69.64091573300004, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 240910.33013297882, "peak_memory_bytes": 6039977472, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "none", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sf-sgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..f7fe52d --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,77 @@ +Starting Experiment: sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.01, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGDScheduleFree ( +Parameter Group 0 + foreach: True + k: 0 + lr: 0.01 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + foreach: True + k: 0 + lr: 0.01 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=10.493 iter_dt=2.69e-01s lr=1.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=10.197 iter_dt=2.67e-01s lr=1.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=9.989 iter_dt=2.67e-01s lr=1.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=9.878 iter_dt=2.68e-01s lr=1.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=9.660 iter_dt=2.66e-01s lr=1.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=9.712 val_pp=16513.055 val_acc=0.037819 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..2808e29 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [10.49294662475586, 10.196949005126953, 9.989242553710938, 9.877887725830078, 9.659662246704102], "val_loss": [9.711913108825684], "val_pp": [16513.055279307664], "val_acc": [0.03781890869140625], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 9.711913108825684, "val_perplexity": 16513.055279307664, "val_accuracy": 0.03781890869140625, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.269430461, 0.266922875, 0.266915019, 0.26828698, 0.265691751], "train_time_seconds": 68.88036811200004, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 243570.35915836148, "peak_memory_bytes": 6039977472, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "none", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.01, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sf-sgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216/run.log new file mode 100644 index 0000000..1f64c5a --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216/run.log @@ -0,0 +1,77 @@ +Starting Experiment: sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 2.43, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGDScheduleFree ( +Parameter Group 0 + foreach: True + k: 0 + lr: 2.43 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + foreach: True + k: 0 + lr: 2.43 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.163 iter_dt=2.68e-01s lr=2.43e+00 +Train: Iter=100 (0.012207 effective passes) train_loss=7.702 iter_dt=2.67e-01s lr=2.43e+00 +Train: Iter=150 (0.018311 effective passes) train_loss=7.607 iter_dt=2.68e-01s lr=2.43e+00 +Train: Iter=200 (0.024414 effective passes) train_loss=7.747 iter_dt=2.68e-01s lr=2.43e+00 +Train: Iter=250 (0.030518 effective passes) train_loss=7.617 iter_dt=2.72e-01s lr=2.43e+00 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.590 val_pp=1977.611 val_acc=0.052864 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216/summary.json new file mode 100644 index 0000000..4b968e3 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.163247108459473, 7.70201301574707, 7.606741905212402, 7.747134208679199, 7.617166519165039], "val_loss": [7.5896501541137695], "val_pp": [1977.611433827244], "val_acc": [0.05286407470703125], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 7.5896501541137695, "val_perplexity": 1977.611433827244, "val_accuracy": 0.05286407470703125, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.268001886, 0.267251018, 0.267919044, 0.267768102, 0.271531302], "train_time_seconds": 69.21735874999996, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 242384.51600842123, "peak_memory_bytes": 6039977472, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "none", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 2.43, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sf-sgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216/run.log new file mode 100644 index 0000000..be1a191 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216/run.log @@ -0,0 +1,77 @@ +Starting Experiment: sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.27, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGDScheduleFree ( +Parameter Group 0 + foreach: True + k: 0 + lr: 0.27 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + foreach: True + k: 0 + lr: 0.27 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.688 iter_dt=2.72e-01s lr=2.70e-01 +Train: Iter=100 (0.012207 effective passes) train_loss=8.044 iter_dt=2.73e-01s lr=2.70e-01 +Train: Iter=150 (0.018311 effective passes) train_loss=7.837 iter_dt=2.73e-01s lr=2.70e-01 +Train: Iter=200 (0.024414 effective passes) train_loss=7.892 iter_dt=2.72e-01s lr=2.70e-01 +Train: Iter=250 (0.030518 effective passes) train_loss=7.677 iter_dt=2.73e-01s lr=2.70e-01 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.712 val_pp=2234.376 val_acc=0.086718 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216/summary.json new file mode 100644 index 0000000..deb7325 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.687780380249023, 8.043517112731934, 7.836560249328613, 7.892111301422119, 7.67655086517334], "val_loss": [7.711722373962402], "val_pp": [2234.3758060934124], "val_acc": [0.08671760559082031], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 7.711722373962402, "val_perplexity": 2234.3758060934124, "val_accuracy": 0.08671760559082031, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.272258159, 0.272763863, 0.272985031, 0.271959269, 0.272847497], "train_time_seconds": 70.50132499499999, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 237970.22256233983, "peak_memory_bytes": 6039977472, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "none", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.27, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sf-sgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..6de19ce --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,77 @@ +Starting Experiment: sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.03, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGDScheduleFree ( +Parameter Group 0 + foreach: True + k: 0 + lr: 0.03 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + foreach: True + k: 0 + lr: 0.03 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=10.092 iter_dt=2.69e-01s lr=3.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=9.505 iter_dt=2.69e-01s lr=3.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=9.150 iter_dt=2.69e-01s lr=3.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=9.079 iter_dt=2.70e-01s lr=3.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=8.863 iter_dt=2.69e-01s lr=3.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=8.906 val_pp=7377.391 val_acc=0.039589 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..fba6fd2 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [10.091880798339844, 9.504928588867188, 9.150369644165039, 9.079030990600586, 8.862791061401367], "val_loss": [8.906181335449219], "val_pp": [7377.391093153199], "val_acc": [0.03958892822265625], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 8.906181335449219, "val_perplexity": 7377.391093153199, "val_accuracy": 0.03958892822265625, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.269387862, 0.269362344, 0.269261945, 0.269575514, 0.269461502], "train_time_seconds": 69.46517621699999, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 241519.80767442673, "peak_memory_bytes": 6039977472, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "none", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.03, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sf-sgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216/run.log new file mode 100644 index 0000000..b49dcc1 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216/run.log @@ -0,0 +1,77 @@ +Starting Experiment: sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 7.29, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGDScheduleFree ( +Parameter Group 0 + foreach: True + k: 0 + lr: 7.29 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + foreach: True + k: 0 + lr: 7.29 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.415 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=100 (0.012207 effective passes) train_loss=7.823 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=150 (0.018311 effective passes) train_loss=7.725 iter_dt=2.73e-01s lr=7.29e+00 +Train: Iter=200 (0.024414 effective passes) train_loss=7.884 iter_dt=2.71e-01s lr=7.29e+00 +Train: Iter=250 (0.030518 effective passes) train_loss=7.629 iter_dt=2.73e-01s lr=7.29e+00 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.577 val_pp=1953.157 val_acc=0.050619 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216/summary.json new file mode 100644 index 0000000..e3a62b8 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.415262222290039, 7.823009967803955, 7.725205421447754, 7.8842854499816895, 7.628593444824219], "val_loss": [7.577207565307617], "val_pp": [1953.1572962701177], "val_acc": [0.05061912536621094], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 7.577207565307617, "val_perplexity": 1953.1572962701177, "val_accuracy": 0.05061912536621094, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.270652709, 0.270780637, 0.272517066, 0.270795019, 0.272634004], "train_time_seconds": 69.78736949199998, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 240404.7626687411, "peak_memory_bytes": 6039977472, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "none", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 7.29, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sf-sgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216/run.log new file mode 100644 index 0000000..39f3ea3 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216/run.log @@ -0,0 +1,77 @@ +Starting Experiment: sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.81, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGDScheduleFree ( +Parameter Group 0 + foreach: True + k: 0 + lr: 0.81 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + foreach: True + k: 0 + lr: 0.81 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.361 iter_dt=2.75e-01s lr=8.10e-01 +Train: Iter=100 (0.012207 effective passes) train_loss=7.833 iter_dt=2.73e-01s lr=8.10e-01 +Train: Iter=150 (0.018311 effective passes) train_loss=7.656 iter_dt=2.73e-01s lr=8.10e-01 +Train: Iter=200 (0.024414 effective passes) train_loss=7.798 iter_dt=2.72e-01s lr=8.10e-01 +Train: Iter=250 (0.030518 effective passes) train_loss=7.625 iter_dt=2.75e-01s lr=8.10e-01 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.600 val_pp=1998.699 val_acc=0.081297 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216/summary.json new file mode 100644 index 0000000..6ae867a --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.36074447631836, 7.832712173461914, 7.655902862548828, 7.7976179122924805, 7.624537944793701], "val_loss": [7.60025691986084], "val_pp": [1998.6991192268076], "val_acc": [0.08129692077636719], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 7.60025691986084, "val_perplexity": 1998.6991192268076, "val_accuracy": 0.08129692077636719, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.275358786, 0.273081391, 0.273341757, 0.271613418, 0.274759312], "train_time_seconds": 70.46290074299996, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 238099.99053532732, "peak_memory_bytes": 6039977472, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "none", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.81, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sf-sgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216/run.log new file mode 100644 index 0000000..d10cceb --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216/run.log @@ -0,0 +1,77 @@ +Starting Experiment: sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'none', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.09, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sf-sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGDScheduleFree ( +Parameter Group 0 + foreach: True + k: 0 + lr: 0.09 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.1 + weight_lr_power: 2.0 + weight_sum: 0.0 + +Parameter Group 1 + foreach: True + k: 0 + lr: 0.09 + lr_max: -1.0 + momentum: 0.9 + r: 0.0 + scheduled_lr: 0.0 + train_mode: False + warmup_steps: 16 + weight_decay: 0.0 + weight_lr_power: 2.0 + weight_sum: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=9.315 iter_dt=2.72e-01s lr=9.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=8.701 iter_dt=2.70e-01s lr=9.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=8.448 iter_dt=2.72e-01s lr=9.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=8.467 iter_dt=2.72e-01s lr=9.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=8.177 iter_dt=2.82e-01s lr=9.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=8.235 val_pp=3768.939 val_acc=0.068001 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216/summary.json new file mode 100644 index 0000000..7aaaa3a --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [9.315433502197266, 8.700555801391602, 8.448097229003906, 8.46670150756836, 8.177438735961914], "val_loss": [8.234554290771484], "val_pp": [3768.938786317613], "val_acc": [0.06800079345703125], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 8.234554290771484, "val_perplexity": 3768.938786317613, "val_accuracy": 0.06800079345703125, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.271592902, 0.270497503, 0.271714379, 0.272013446, 0.281665016], "train_time_seconds": 70.36729345900002, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 238423.494428919, "peak_memory_bytes": 6039977472, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "none", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.09, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sf-sgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..e47db2d --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,71 @@ +Starting Experiment: sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGD ( +Parameter Group 0 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.003 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.003 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=9.957 iter_dt=2.77e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=9.120 iter_dt=2.75e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=8.751 iter_dt=2.76e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=8.698 iter_dt=2.79e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=8.428 iter_dt=2.75e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=8.424 val_pp=4556.842 val_acc=0.058386 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..701269e --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [9.956962585449219, 9.119741439819336, 8.751251220703125, 8.698186874389648, 8.427891731262207], "val_loss": [8.42439079284668], "val_pp": [4556.842033847346], "val_acc": [0.05838584899902344], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 8.42439079284668, "val_perplexity": 4556.842033847346, "val_accuracy": 0.05838584899902344, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.2768191, 0.275133483, 0.27624556, 0.278918495, 0.275351877], "train_time_seconds": 71.36861597700005, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 235078.3431950928, "peak_memory_bytes": 6039715328, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..12812ea --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,71 @@ +Starting Experiment: sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.01, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGD ( +Parameter Group 0 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.01 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.01 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=9.138 iter_dt=2.74e-01s lr=1.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=8.395 iter_dt=2.73e-01s lr=1.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=8.102 iter_dt=2.69e-01s lr=1.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=8.111 iter_dt=2.73e-01s lr=1.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=7.845 iter_dt=2.73e-01s lr=1.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.847 val_pp=2558.660 val_acc=0.078848 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..61ae27f --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [9.138038635253906, 8.39537239074707, 8.102432250976562, 8.110796928405762, 7.8445820808410645], "val_loss": [7.8472442626953125], "val_pp": [2558.6600544809194], "val_acc": [0.07884788513183594], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 7.8472442626953125, "val_perplexity": 2558.6600544809194, "val_accuracy": 0.07884788513183594, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.274096104, 0.272603517, 0.269231601, 0.273377931, 0.2729751], "train_time_seconds": 70.252648892, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 238812.57524953628, "peak_memory_bytes": 6039715328, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.01, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216/run.log new file mode 100644 index 0000000..6e65c3c --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216/run.log @@ -0,0 +1,71 @@ +Starting Experiment: sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.27, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGD ( +Parameter Group 0 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.27 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.27 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.297 iter_dt=2.72e-01s lr=2.70e-01 +Train: Iter=100 (0.012207 effective passes) train_loss=7.694 iter_dt=2.73e-01s lr=2.70e-01 +Train: Iter=150 (0.018311 effective passes) train_loss=7.655 iter_dt=2.73e-01s lr=2.70e-01 +Train: Iter=200 (0.024414 effective passes) train_loss=7.684 iter_dt=2.76e-01s lr=2.70e-01 +Train: Iter=250 (0.030518 effective passes) train_loss=7.576 iter_dt=2.75e-01s lr=2.70e-01 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.624 val_pp=2046.617 val_acc=0.066933 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216/summary.json new file mode 100644 index 0000000..46464ee --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.296594619750977, 7.694429397583008, 7.6546525955200195, 7.684326171875, 7.575793266296387], "val_loss": [7.623948574066162], "val_pp": [2046.616960538731], "val_acc": [0.06693267822265625], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 7.623948574066162, "val_perplexity": 2046.616960538731, "val_accuracy": 0.06693267822265625, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.272050949, 0.273129246, 0.272948321, 0.276206282, 0.274562448], "train_time_seconds": 70.37890287699994, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 238384.1650575495, "peak_memory_bytes": 6039715328, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.27, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..5925c28 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,71 @@ +Starting Experiment: sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.03, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGD ( +Parameter Group 0 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.03 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.03 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.677 iter_dt=2.70e-01s lr=3.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=7.972 iter_dt=2.70e-01s lr=3.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=7.785 iter_dt=2.84e-01s lr=3.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=7.890 iter_dt=2.76e-01s lr=3.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=7.716 iter_dt=2.73e-01s lr=3.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.710 val_pp=2230.875 val_acc=0.053326 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..383da92 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.67727279663086, 7.972423553466797, 7.785012245178223, 7.890357971191406, 7.716372489929199], "val_loss": [7.7101545333862305], "val_pp": [2230.875408146757], "val_acc": [0.053325653076171875], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 7.7101545333862305, "val_perplexity": 2230.875408146757, "val_accuracy": 0.053325653076171875, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.269811265, 0.270352124, 0.28355507, 0.276152641, 0.272560996], "train_time_seconds": 70.45152964899997, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 238138.42060756657, "peak_memory_bytes": 6039715328, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.03, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216/run.log new file mode 100644 index 0000000..a41b656 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216/run.log @@ -0,0 +1,71 @@ +Starting Experiment: sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.09, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SGD ( +Parameter Group 0 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.09 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0 + differentiable: False + foreach: None + fused: None + lr: 0.09 + maximize: False + momentum: 0.9 + nesterov: False + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.301 iter_dt=2.73e-01s lr=9.00e-02 +Train: Iter=100 (0.012207 effective passes) train_loss=7.720 iter_dt=2.70e-01s lr=9.00e-02 +Train: Iter=150 (0.018311 effective passes) train_loss=7.597 iter_dt=2.72e-01s lr=9.00e-02 +Train: Iter=200 (0.024414 effective passes) train_loss=7.730 iter_dt=2.71e-01s lr=9.00e-02 +Train: Iter=250 (0.030518 effective passes) train_loss=7.597 iter_dt=2.71e-01s lr=9.00e-02 +>Eval: Iter=256 (0.031250 effective passes) val_loss=7.590 val_pp=1978.672 val_acc=0.047283 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216/summary.json new file mode 100644 index 0000000..32e794c --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.301076889038086, 7.719658851623535, 7.597330093383789, 7.730240821838379, 7.596835136413574], "val_loss": [7.59018611907959], "val_pp": [1978.6716476519782], "val_acc": [0.047283172607421875], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 7.59018611907959, "val_perplexity": 1978.6716476519782, "val_accuracy": 0.047283172607421875, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.27284039, 0.269898792, 0.271540275, 0.271442091, 0.271066566], "train_time_seconds": 70.11675072299994, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 239275.43457168047, "peak_memory_bytes": 6039715328, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.09, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216/run.log new file mode 100644 index 0000000..d2a0471 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216/run.log @@ -0,0 +1,65 @@ +Starting Experiment: sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'signsgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Signum ( +Parameter Group 0 + dampening: 0.0 + lr: 0.0001 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0.0 + lr: 0.0001 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.674 iter_dt=2.72e-01s lr=1.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=7.313 iter_dt=2.75e-01s lr=1.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.895 iter_dt=2.74e-01s lr=1.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.888 iter_dt=2.70e-01s lr=1.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.720 iter_dt=2.76e-01s lr=1.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.662 val_pp=781.975 val_acc=0.146978 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216/summary.json new file mode 100644 index 0000000..43deb43 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.673837661743164, 7.312655448913574, 6.895467281341553, 6.887711524963379, 6.720067024230957], "val_loss": [6.661827087402344], "val_pp": [781.9748714287975], "val_acc": [0.14697837829589844], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.661827087402344, "val_perplexity": 781.9748714287975, "val_accuracy": 0.14697837829589844, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.272289696, 0.274695333, 0.274248289, 0.269570843, 0.276418062], "train_time_seconds": 70.893569206, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 236653.56657737694, "peak_memory_bytes": 5909968384, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "signsgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..e8e2040 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,65 @@ +Starting Experiment: sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'signsgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Signum ( +Parameter Group 0 + dampening: 0.0 + lr: 0.0003 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0.0 + lr: 0.0003 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.497 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.773 iter_dt=2.68e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.607 iter_dt=2.62e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.665 iter_dt=2.69e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.565 iter_dt=2.67e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.476 val_pp=649.133 val_acc=0.151705 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..eb8058a --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.4969282150268555, 6.772626876831055, 6.606821060180664, 6.6653008460998535, 6.564878463745117], "val_loss": [6.475642204284668], "val_pp": [649.1331441433161], "val_acc": [0.1517047882080078], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.475642204284668, "val_perplexity": 649.1331441433161, "val_accuracy": 0.1517047882080078, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.268542218, 0.26751371, 0.261522974, 0.268884907, 0.267349068], "train_time_seconds": 68.95654871499998, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 243301.27178117435, "peak_memory_bytes": 5909968384, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "signsgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..fadbc18 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,65 @@ +Starting Experiment: sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'signsgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Signum ( +Parameter Group 0 + dampening: 0.0 + lr: 0.001 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0.0 + lr: 0.001 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.822 iter_dt=2.72e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.048 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.855 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.814 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.840 iter_dt=2.69e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.683 val_pp=798.798 val_acc=0.133631 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..cb86376 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.822054862976074, 7.04815149307251, 6.855133056640625, 6.814094066619873, 6.8399338722229], "val_loss": [6.683113098144531], "val_pp": [798.7984033511141], "val_acc": [0.13363075256347656], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.683113098144531, "val_perplexity": 798.7984033511141, "val_accuracy": 0.13363075256347656, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.272294521, 0.269452719, 0.268879901, 0.268759892, 0.268970109], "train_time_seconds": 69.328829448, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 241994.79687715962, "peak_memory_bytes": 5909968384, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "signsgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..d6e714a --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,65 @@ +Starting Experiment: sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'signsgd', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Signum ( +Parameter Group 0 + dampening: 0.0 + lr: 0.003 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0.0 + lr: 0.003 + momentum: 0.0 + nesterov: False + sign_update: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.846 iter_dt=2.71e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.349 iter_dt=2.71e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=7.010 iter_dt=2.70e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=7.113 iter_dt=2.83e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.960 iter_dt=2.73e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.855 val_pp=948.329 val_acc=0.114681 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..99f6d69 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.845541000366211, 7.349099636077881, 7.010373115539551, 7.112664222717285, 6.959585666656494], "val_loss": [6.854705810546875], "val_pp": [948.3287261686504], "val_acc": [0.11468124389648438], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.854705810546875, "val_perplexity": 948.3287261686504, "val_accuracy": 0.11468124389648438, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.271237939, 0.271035474, 0.270397545, 0.283066704, 0.273045782], "train_time_seconds": 70.11524923599995, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 239280.5585491082, "peak_memory_bytes": 5909968384, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "signsgd", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..22a5296 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,65 @@ +Starting Experiment: sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'signum', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Signum ( +Parameter Group 0 + dampening: 0.0 + lr: 0.0003 + momentum: 0.9 + nesterov: False + sign_update: True + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0.0 + lr: 0.0003 + momentum: 0.9 + nesterov: False + sign_update: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.219 iter_dt=2.75e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=7.310 iter_dt=2.73e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.811 iter_dt=2.77e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.641 iter_dt=2.74e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.399 iter_dt=2.73e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.296 val_pp=542.439 val_acc=0.146202 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..d4280b9 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.21932601928711, 7.309566497802734, 6.810800075531006, 6.64058256149292, 6.398901462554932], "val_loss": [6.296080112457275], "val_pp": [542.4391305573229], "val_acc": [0.14620208740234375], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.296080112457275, "val_perplexity": 542.4391305573229, "val_accuracy": 0.14620208740234375, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.275088288, 0.27315078, 0.27658288, 0.273646449, 0.273248824], "train_time_seconds": 70.628779494, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 237540.7888993076, "peak_memory_bytes": 6040567296, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "signum", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..79f2fb6 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,65 @@ +Starting Experiment: sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'signum', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Signum ( +Parameter Group 0 + dampening: 0.0 + lr: 0.001 + momentum: 0.9 + nesterov: False + sign_update: True + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0.0 + lr: 0.001 + momentum: 0.9 + nesterov: False + sign_update: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.903 iter_dt=2.67e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.397 iter_dt=2.69e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.743 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.568 iter_dt=2.76e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.318 iter_dt=2.80e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.220 val_pp=502.641 val_acc=0.149128 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..1ac1532 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.903072357177734, 7.397322654724121, 6.743289470672607, 6.568361282348633, 6.318418025970459], "val_loss": [6.2198805809021], "val_pp": [502.641100274047], "val_acc": [0.14912796020507812], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.2198805809021, "val_perplexity": 502.641100274047, "val_accuracy": 0.14912796020507812, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.267070114, 0.268802736, 0.276310311, 0.276065951, 0.280459211], "train_time_seconds": 70.13492149399995, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 239213.44235674792, "peak_memory_bytes": 6040567296, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "signum", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..8d844bd --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,65 @@ +Starting Experiment: sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'signum', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.95, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +Signum ( +Parameter Group 0 + dampening: 0.0 + lr: 0.003 + momentum: 0.9 + nesterov: False + sign_update: True + weight_decay: 0.1 + +Parameter Group 1 + dampening: 0.0 + lr: 0.003 + momentum: 0.9 + nesterov: False + sign_update: True + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.135 iter_dt=2.69e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.197 iter_dt=2.70e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.662 iter_dt=2.70e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.599 iter_dt=2.69e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.466 iter_dt=2.70e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.385 val_pp=592.681 val_acc=0.139008 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..8c30f0e --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.13547134399414, 7.196522235870361, 6.662273406982422, 6.5992841720581055, 6.465832710266113], "val_loss": [6.384660720825195], "val_pp": [592.6810679067656], "val_acc": [0.139007568359375], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.384660720825195, "val_perplexity": 592.6810679067656, "val_accuracy": 0.139007568359375, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.268688617, 0.269870708, 0.269530374, 0.269373426, 0.270497469], "train_time_seconds": 69.51772536999995, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 241337.24040459082, "peak_memory_bytes": 6040567296, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "signum", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.95, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..834d788 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,75 @@ +Starting Experiment: sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'soap', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SOAP ( +Parameter Group 0 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.0003 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.0003 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.838 iter_dt=2.78e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=7.767 iter_dt=2.83e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=7.085 iter_dt=2.84e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.685 iter_dt=2.85e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.351 iter_dt=2.87e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.225 val_pp=505.345 val_acc=0.183193 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..120abee --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.837679862976074, 7.766672611236572, 7.085450172424316, 6.68472957611084, 6.35131311416626], "val_loss": [6.225244998931885], "val_pp": [505.3447206276595], "val_acc": [0.18319320678710938], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.225244998931885, "val_perplexity": 505.3447206276595, "val_accuracy": 0.18319320678710938, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.278261155, 0.283015515, 0.283925757, 0.284594022, 0.286650502], "train_time_seconds": 81.71295656799995, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 205318.92009118924, "peak_memory_bytes": 6556407296, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "soap", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..29fa50c --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,75 @@ +Starting Experiment: sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'soap', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SOAP ( +Parameter Group 0 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.001 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.001 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.381 iter_dt=2.97e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.149 iter_dt=2.91e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=5.709 iter_dt=2.89e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.569 iter_dt=2.86e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.440 iter_dt=2.86e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.292 val_pp=198.724 val_acc=0.209654 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..9a720e4 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.380842685699463, 6.149449348449707, 5.709247589111328, 5.569197654724121, 5.439713954925537], "val_loss": [5.291918754577637], "val_pp": [198.723655728276], "val_acc": [0.2096538543701172], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.291918754577637, "val_perplexity": 198.723655728276, "val_accuracy": 0.2096538543701172, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.297197397, 0.290959468, 0.289061069, 0.28614129, 0.285736941], "train_time_seconds": 83.30477875699998, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 201395.60119280955, "peak_memory_bytes": 6556407296, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "soap", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..ff55797 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,75 @@ +Starting Experiment: sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'soap', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SOAP ( +Parameter Group 0 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.003 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + correct_bias: True + eps: 1e-08 + lr: 0.003 + max_precond_dim: 10000 + merge_dims: False + normalize_grads: False + precondition_1d: False + precondition_frequency: 10 + shampoo_beta: -1.0 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.442 iter_dt=2.83e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.490 iter_dt=2.84e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.093 iter_dt=2.82e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=5.905 iter_dt=2.85e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=5.732 iter_dt=2.83e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=5.588 val_pp=267.297 val_acc=0.185127 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..4a2440f --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.442171096801758, 6.489656925201416, 6.09266471862793, 5.904678821563721, 5.7318572998046875], "val_loss": [5.5883636474609375], "val_pp": [267.29686332990445], "val_acc": [0.18512725830078125], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 5.5883636474609375, "val_perplexity": 267.29686332990445, "val_accuracy": 0.18512725830078125, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.283212594, 0.283944155, 0.281821348, 0.284712435, 0.283187009], "train_time_seconds": 82.14935480000007, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 204228.21385323876, "peak_memory_bytes": 6556407296, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "soap", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216/run.log new file mode 100644 index 0000000..807c09f --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216/run.log @@ -0,0 +1,65 @@ +Starting Experiment: sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sophiag', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SophiaG ( +Parameter Group 0 + betas: (0.9, 0.999) + capturable: False + lr: 0.0001 + maximize: False + rho: 0.04 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + capturable: False + lr: 0.0001 + maximize: False + rho: 0.04 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.622 iter_dt=3.23e-01s lr=1.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=7.197 iter_dt=3.24e-01s lr=1.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.638 iter_dt=3.30e-01s lr=1.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.449 iter_dt=3.23e-01s lr=1.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.202 iter_dt=3.26e-01s lr=1.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.111 val_pp=450.936 val_acc=0.176609 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216/summary.json new file mode 100644 index 0000000..97dc134 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.622295379638672, 7.197116851806641, 6.637634754180908, 6.449067115783691, 6.201791763305664], "val_loss": [6.111329078674316], "val_pp": [450.935794965251], "val_acc": [0.17660903930664062], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.111329078674316, "val_perplexity": 450.935794965251, "val_accuracy": 0.17660903930664062, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.322547516, 0.324060032, 0.330104769, 0.322802625, 0.326217938], "train_time_seconds": 72.79379595199997, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 230475.90499419553, "peak_memory_bytes": 14279713792, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sophiag", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216/run.log new file mode 100644 index 0000000..2b2c874 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216/run.log @@ -0,0 +1,65 @@ +Starting Experiment: sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.0003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sophiag', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SophiaG ( +Parameter Group 0 + betas: (0.9, 0.999) + capturable: False + lr: 0.0003 + maximize: False + rho: 0.04 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + capturable: False + lr: 0.0003 + maximize: False + rho: 0.04 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.701 iter_dt=3.24e-01s lr=3.00e-04 +Train: Iter=100 (0.012207 effective passes) train_loss=6.784 iter_dt=3.20e-01s lr=3.00e-04 +Train: Iter=150 (0.018311 effective passes) train_loss=6.428 iter_dt=3.22e-01s lr=3.00e-04 +Train: Iter=200 (0.024414 effective passes) train_loss=6.336 iter_dt=3.22e-01s lr=3.00e-04 +Train: Iter=250 (0.030518 effective passes) train_loss=6.145 iter_dt=3.22e-01s lr=3.00e-04 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.051 val_pp=424.420 val_acc=0.160858 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216/summary.json new file mode 100644 index 0000000..00fc65b --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.701247692108154, 6.783934116363525, 6.428488254547119, 6.336462020874023, 6.144689083099365], "val_loss": [6.050726890563965], "val_pp": [424.4196984680204], "val_acc": [0.160858154296875], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.050726890563965, "val_perplexity": 424.4196984680204, "val_accuracy": 0.160858154296875, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.32388024, 0.320227945, 0.321704709, 0.322493764, 0.321796907], "train_time_seconds": 72.48078954399999, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 231471.209206617, "peak_memory_bytes": 14279713792, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.0003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sophiag", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216/run.log new file mode 100644 index 0000000..4adb797 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216/run.log @@ -0,0 +1,65 @@ +Starting Experiment: sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.001, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sophiag', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SophiaG ( +Parameter Group 0 + betas: (0.9, 0.999) + capturable: False + lr: 0.001 + maximize: False + rho: 0.04 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + capturable: False + lr: 0.001 + maximize: False + rho: 0.04 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=7.601 iter_dt=3.21e-01s lr=1.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=6.793 iter_dt=3.22e-01s lr=1.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=6.489 iter_dt=3.24e-01s lr=1.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=6.454 iter_dt=3.23e-01s lr=1.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=6.287 iter_dt=3.23e-01s lr=1.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.201 val_pp=493.297 val_acc=0.148426 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216/summary.json new file mode 100644 index 0000000..a7e5d7c --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [7.600943565368652, 6.7925801277160645, 6.488624572753906, 6.454020023345947, 6.287290573120117], "val_loss": [6.201114654541016], "val_pp": [493.29653463131854], "val_acc": [0.14842605590820312], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.201114654541016, "val_perplexity": 493.29653463131854, "val_accuracy": 0.14842605590820312, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.321158706, 0.321938208, 0.324063634, 0.323116587, 0.322565966], "train_time_seconds": 72.68963629099997, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 230806.16241956988, "peak_memory_bytes": 14279713792, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.001, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sophiag", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216/run.log b/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216/run.log new file mode 100644 index 0000000..41a44a0 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216/run.log @@ -0,0 +1,65 @@ +Starting Experiment: sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216 +Experiment Directory: /root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216 +Config: +{'config_format': 'base', 'run_prefix': None, 'experiment_name': 'sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216', 'seed': 0, 'data_seed': 1337, 'eval_interval': 0, 'full_eval_at': [], 'eval_batches': 64, 'device': 'cuda:0', 'distributed_backend': None, 'log_interval': 50, 'results_base_folder': '/root/autodl-tmp/llm-optimizer-results/formal', 'permanent_ckpt_interval': 0, 'latest_ckpt_interval': 256, 'resume_from': None, 'resume_from_swa': None, 'auto_resume': True, 'wandb': False, 'wandb_project': 'my-project', 'wandb_run_prefix': 'none', 'eval_seq_prefix': 'none', 'log_dynamics': False, 'dynamics_logger_cfg': './src/logger/rotational_logger.yaml', 'wandb_entity': None, 'log_parameter_norms': False, 'norm_order': 2, 'scheduler': 'warmup_constant', 'final_div_factor': 1, 'cos_inf_steps': 0, 'iterations': 15000, 'warmup_steps': 16, 'lr': 0.003, 'train_token_budget': 16777216, 'eval_at_tokens': [16777216], 'strict_sub_one_pass': True, 'fixed_data_boundaries': True, 'lazy_data_permutation': True, 'limit_final_eval': True, 'wsd_final_lr_scale': 0.0, 'wsd_fract_decay': 0.1, 'decay_type': 'linear', 'opt': 'sophiag', 'batch_size': 16, 'acc_steps': 8, 'weight_decay': 0.1, 'beta1': 0.9, 'beta2': 0.999, 'grad_clip': 0.5, 'momentum': 0.9, 'shampoo_beta': -1.0, 'precondition_frequency': 10, 'max_precond_dim': 10000, 'merge_dims': False, 'precondition_1d': False, 'normalize_grads': False, 'soap_data_format': 'channels_first', 'correct_bias': True, 'nesterov': False, 'muon_ns_steps': 5, 'muon_lr_factor': 1.0, 'muon_adamw_lr': 0.001, 'adema_beta3': 0.9, 'adema_alpha': 2.0, 'adema_beta3_warmup': None, 'adema_alpha_warmup': None, 'schedulefree_r': 0.0, 'weight_lr_power': 2.0, 'dampening': 0.0, 'prodigy_beta3': None, 'prodigy_decouple': True, 'prodigy_use_bias_correction': False, 'prodigy_safeguard_warmup': False, 'prodigy_fsdp_in_use': False, 'sophia_rho': 0.04, 'sophia_bs': 480, 'clipping_type': 'no', 'clip_eta': 1.0, 'mars_type': 'mars-adamw', 'mars_vr_gamma': 0.025, 'mars_is_approx': True, 'mars_lr': 0.003, 'mars_beta1': 0.95, 'mars_beta2': 0.99, 'adafactor_decay_rate': -0.8, 'lamb_use_bias_correction': False, 'adopt_decouple': True, 'adopt_eps': 1e-06, 'scion_lmh_scale': 10.0, 'scion_emb_scale': 1.0, 'scion_tr_scale': 3.0, 'weight_average': False, 'wa_interval': 5, 'wa_horizon': 500, 'wa_dtype': 'float32', 'wa_use_temp_dir': False, 'wa_sweep_horizon': False, 'max_num_wa_sweeps': 5, 'exponential_weight_average': False, 'ewa_interval': 10, 'ewa_decay': 0.95, 'ewa_after_warmup': False, 'datasets_dir': '/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets', 'dataset': 'token-bin', 'tokenizer': 'gpt2', 'vocab_size': 50304, 'data_in_ram': False, 'train_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin', 'val_data_path': '/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin', 'model': 'llama', 'parallel_block': False, 'use_pretrained': 'none', 'from_dense': False, 'init_std': 0.02, 'dropout': 0.0, 'n_head': 6, 'n_layer': 8, 'sequence_length': 512, 'n_embd': 384, 'multiple_of': 256, 'n_kv_head': None, 'rmsnorm_eps': 1e-05, 'dtype': 'bfloat16', 'bias': False, 'compile': False, 'untied_embeds': False, 'mlp_dim_exp_factor': 1.0, 'moe': False, 'moe_routing': 'standard_gating', 'moe_num_experts': 8, 'capacity_factor': 2.0, 'moe_num_shared_experts': 0, 'moe_router_loss': 'load_balancing_z_loss', 'moe_num_experts_per_tok': 2, 'moe_entropy_loss_factor': 0.01, 'moe_aux_loss_factor': 0.1, 'moe_z_loss_factor': 0.01, 'moe_softmax_order': 'topk_softmax', 'plot_router_logits': False, 'scale_emb': 10, 'scale_base_model': 256, 'scale_depth': 1.4, 'world_size': 1} + +Loading dataset: 'token-bin' +Num training tokens: 536870912 +Num validation tokens: 8388608 +Token budget: 16,777,216 tokens, 256 optimizer steps, 0.031250 effective passes +Evaluation token boundaries: 16,777,216 + +Model: +Llama( + (transformer): ModuleDict( + (wte): Embedding(50304, 384) + (drop): Dropout(p=0.0, inplace=False) + (h): ModuleList( + (0-7): 8 x LlamaBlock( + (ln_1): RMSNorm() + (attn): LlamaAttention( + (c_attn): Linear(in_features=384, out_features=1152, bias=False) + (c_proj): Linear(in_features=384, out_features=384, bias=False) + (attn_dropout): Dropout(p=0.0, inplace=False) + (resid_dropout): Dropout(p=0.0, inplace=False) + ) + (ln_2): RMSNorm() + (mlp): LlamaMLP( + (w1): Linear(in_features=384, out_features=1024, bias=False) + (w2): Linear(in_features=384, out_features=1024, bias=False) + (c_proj): Linear(in_features=1024, out_features=384, bias=False) + ) + ) + ) + (ln_f): RMSNorm() + ) + (lm_head): Linear(in_features=384, out_features=50304, bias=False) +) +number of parameters: 33.48M +number of optimized parameters: 33.48M +number of non-embedding parameters: -5.15M + +Optimizer: +SophiaG ( +Parameter Group 0 + betas: (0.9, 0.999) + capturable: False + lr: 0.003 + maximize: False + rho: 0.04 + weight_decay: 0.1 + +Parameter Group 1 + betas: (0.9, 0.999) + capturable: False + lr: 0.003 + maximize: False + rho: 0.04 + weight_decay: 0.0 +) +Train: Iter=50 (0.006104 effective passes) train_loss=8.333 iter_dt=3.17e-01s lr=3.00e-03 +Train: Iter=100 (0.012207 effective passes) train_loss=7.600 iter_dt=3.18e-01s lr=3.00e-03 +Train: Iter=150 (0.018311 effective passes) train_loss=7.281 iter_dt=3.18e-01s lr=3.00e-03 +Train: Iter=200 (0.024414 effective passes) train_loss=7.218 iter_dt=3.20e-01s lr=3.00e-03 +Train: Iter=250 (0.030518 effective passes) train_loss=7.025 iter_dt=3.19e-01s lr=3.00e-03 +>Eval: Iter=256 (0.031250 effective passes) val_loss=6.951 val_pp=1044.170 val_acc=0.098793 diff --git a/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216/summary.json b/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216/summary.json new file mode 100644 index 0000000..a104994 --- /dev/null +++ b/exps/sub_one_pass/formal/sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216/summary.json @@ -0,0 +1 @@ +{"train_loss": [8.332549095153809, 7.600278854370117, 7.280983924865723, 7.218073844909668, 7.024627685546875], "val_loss": [6.950982093811035], "val_pp": [1044.1698201716288], "val_acc": [0.09879302978515625], "eval_history": [{"iter": 256, "tokens": 16777216, "data_exposure": 0.03125047684443427, "val_loss": 6.950982093811035, "val_perplexity": 1044.1698201716288, "val_accuracy": 0.09879302978515625, "full_eval": false, "num_eval_batches": 64}], "iteration_time_seconds": [0.317070364, 0.318223464, 0.317587601, 0.32037479, 0.318777886], "train_time_seconds": 71.69533521800004, "processed_tokens_this_run": 16777216, "mean_tokens_per_second": 234007.0793864963, "peak_memory_bytes": 14279713792, "args": {"config_format": "base", "run_prefix": null, "experiment_name": "sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216", "seed": 0, "data_seed": 1337, "eval_interval": 0, "full_eval_at": [], "eval_batches": 64, "device": "cuda:0", "distributed_backend": null, "log_interval": 50, "results_base_folder": "/root/autodl-tmp/llm-optimizer-results/formal", "permanent_ckpt_interval": 0, "latest_ckpt_interval": 256, "resume_from": null, "resume_from_swa": null, "auto_resume": true, "wandb": false, "wandb_project": "my-project", "wandb_run_prefix": "none", "eval_seq_prefix": "none", "log_dynamics": false, "dynamics_logger_cfg": "./src/logger/rotational_logger.yaml", "wandb_entity": null, "log_parameter_norms": false, "norm_order": 2, "scheduler": "warmup_constant", "final_div_factor": 1, "cos_inf_steps": 0, "iterations": 256, "warmup_steps": 16, "lr": 0.003, "train_token_budget": 16777216, "eval_at_tokens": [16777216], "strict_sub_one_pass": true, "fixed_data_boundaries": true, "lazy_data_permutation": true, "limit_final_eval": true, "wsd_final_lr_scale": 0.0, "wsd_fract_decay": 0.1, "decay_type": "linear", "opt": "sophiag", "batch_size": 16, "acc_steps": 8, "weight_decay": 0.1, "beta1": 0.9, "beta2": 0.999, "grad_clip": 0.5, "momentum": 0.9, "shampoo_beta": -1.0, "precondition_frequency": 10, "max_precond_dim": 10000, "merge_dims": false, "precondition_1d": false, "normalize_grads": false, "soap_data_format": "channels_first", "correct_bias": true, "nesterov": false, "muon_ns_steps": 5, "muon_lr_factor": 1.0, "muon_adamw_lr": 0.001, "adema_beta3": 0.9, "adema_alpha": 2.0, "adema_beta3_warmup": null, "adema_alpha_warmup": null, "schedulefree_r": 0.0, "weight_lr_power": 2.0, "dampening": 0.0, "prodigy_beta3": null, "prodigy_decouple": true, "prodigy_use_bias_correction": false, "prodigy_safeguard_warmup": false, "prodigy_fsdp_in_use": false, "sophia_rho": 0.04, "sophia_bs": 480, "clipping_type": "no", "clip_eta": 1.0, "mars_type": "mars-adamw", "mars_vr_gamma": 0.025, "mars_is_approx": true, "mars_lr": 0.003, "mars_beta1": 0.95, "mars_beta2": 0.99, "adafactor_decay_rate": -0.8, "lamb_use_bias_correction": false, "adopt_decouple": true, "adopt_eps": 1e-06, "scion_lmh_scale": 10.0, "scion_emb_scale": 1.0, "scion_tr_scale": 3.0, "weight_average": false, "wa_interval": 5, "wa_horizon": 500, "wa_dtype": "float32", "wa_use_temp_dir": false, "wa_sweep_horizon": false, "max_num_wa_sweeps": 5, "exponential_weight_average": false, "ewa_interval": 10, "ewa_decay": 0.95, "ewa_after_warmup": false, "datasets_dir": "/root/autodl-tmp/llm-optimizer-benchmark/src/data/datasets", "dataset": "token-bin", "tokenizer": "gpt2", "vocab_size": 50304, "data_in_ram": false, "train_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/train.bin", "val_data_path": "/root/autodl-tmp/llm-optimizer-data/fineweb-512m/val.bin", "model": "llama", "parallel_block": false, "use_pretrained": "none", "from_dense": false, "init_std": 0.02, "dropout": 0.0, "n_head": 6, "n_layer": 8, "sequence_length": 512, "n_embd": 384, "multiple_of": 256, "n_kv_head": null, "rmsnorm_eps": 1e-05, "dtype": "bfloat16", "bias": false, "compile": false, "untied_embeds": false, "mlp_dim_exp_factor": 1.0, "moe": false, "moe_routing": "standard_gating", "moe_num_experts": 8, "capacity_factor": 2.0, "moe_num_shared_experts": 0, "moe_router_loss": "load_balancing_z_loss", "moe_num_experts_per_tok": 2, "moe_entropy_loss_factor": 0.01, "moe_aux_loss_factor": 0.1, "moe_z_loss_factor": 0.01, "moe_softmax_order": "topk_softmax", "plot_router_logits": false, "scale_emb": 10, "scale_base_model": 256, "scale_depth": 1.4, "world_size": 1, "tokens_per_iteration": 65536, "data_unique_tokens": 536862720, "eval_at_steps": [256], "token_budget_plan": {"train_token_budget": 16777216, "tokens_per_iteration": 65536, "iterations": 256, "eval_at_tokens": [16777216], "eval_at_steps": [256], "data_unique_tokens": 536862720, "target_data_exposure": 0.03125047684443427}}} \ No newline at end of file diff --git a/exps/sub_one_pass/formal/sub_one_pass_results.csv b/exps/sub_one_pass/formal/sub_one_pass_results.csv new file mode 100644 index 0000000..f2a77c0 --- /dev/null +++ b/exps/sub_one_pass/formal/sub_one_pass_results.csv @@ -0,0 +1,223 @@ +optimizer,seed,data_seed,tokens,data_exposure,iteration,val_loss,val_perplexity,val_accuracy,num_eval_batches,train_time_seconds,mean_tokens_per_second,peak_memory_bytes,experiment_dir +adamw,1,1337,16777216,0.03125047684443427,256,5.624117851257324,277.02674896318183,0.18595123291015625,64,1110.7638108969995,241667.4484409286,6175430656,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456 +adamw,1,1337,33554432,0.06250095368886854,512,5.092065811157227,162.72511822087296,0.21978759765625,64,1110.7638108969995,241667.4484409286,6175430656,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456 +adamw,1,1337,67108864,0.1250019073777371,1024,4.568408966064453,96.39033074510401,0.26802825927734375,64,1110.7638108969995,241667.4484409286,6175430656,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456 +adamw,1,1337,134217728,0.2500038147554742,2048,4.286713123321533,72.72682072742225,0.29187965393066406,64,1110.7638108969995,241667.4484409286,6175430656,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456 +adamw,1,1337,268435456,0.5000076295109483,4096,4.097991943359375,60.2190764599356,0.30730628967285156,64,1110.7638108969995,241667.4484409286,6175430656,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_adamw_seed1_lrscale1_tokens268435456 +adamw,2,1337,16777216,0.03125047684443427,256,5.799754619598389,330.21723256127666,0.17535972595214844,64,1117.7748976870016,240151.62315370503,6175430656,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456 +adamw,2,1337,33554432,0.06250095368886854,512,5.137753486633301,170.33209487378295,0.21564865112304688,64,1117.7748976870016,240151.62315370503,6175430656,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456 +adamw,2,1337,67108864,0.1250019073777371,1024,4.594059467315674,98.89477213389124,0.2672157287597656,64,1117.7748976870016,240151.62315370503,6175430656,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456 +adamw,2,1337,134217728,0.2500038147554742,2048,4.306008338928223,74.14372523990149,0.2904777526855469,64,1117.7748976870016,240151.62315370503,6175430656,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456 +adamw,2,1337,268435456,0.5000076295109483,4096,4.108890533447266,60.87896846029916,0.30696868896484375,64,1117.7748976870016,240151.62315370503,6175430656,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_adamw_seed2_lrscale1_tokens268435456 +ademamix,1,1337,16777216,0.03125047684443427,256,5.652766227722168,285.0778854953414,0.1837139129638672,64,1128.150925535,237942.85846346364,6314994176,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456 +ademamix,1,1337,33554432,0.06250095368886854,512,5.038692951202393,154.26772420847368,0.2230205535888672,64,1128.150925535,237942.85846346364,6314994176,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456 +ademamix,1,1337,67108864,0.1250019073777371,1024,4.498805046081543,89.90935714143036,0.2743968963623047,64,1128.150925535,237942.85846346364,6314994176,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456 +ademamix,1,1337,134217728,0.2500038147554742,2048,4.204828262329102,67.00889916573766,0.30016136169433594,64,1128.150925535,237942.85846346364,6314994176,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456 +ademamix,1,1337,268435456,0.5000076295109483,4096,4.019683837890625,55.68334747159574,0.3166980743408203,64,1128.150925535,237942.85846346364,6314994176,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_ademamix_seed1_lrscale1_tokens268435456 +ademamix,2,1337,16777216,0.03125047684443427,256,5.664737701416016,288.51119547894774,0.18284988403320312,64,1123.6289774160014,238900.43901975403,6314994176,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456 +ademamix,2,1337,33554432,0.06250095368886854,512,4.975729465484619,144.85396812324217,0.23189353942871094,64,1123.6289774160014,238900.43901975403,6314994176,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456 +ademamix,2,1337,67108864,0.1250019073777371,1024,4.477364540100098,88.00217485099525,0.27665138244628906,64,1123.6289774160014,238900.43901975403,6314994176,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456 +ademamix,2,1337,134217728,0.2500038147554742,2048,4.195748329162598,66.40321718440944,0.3009834289550781,64,1123.6289774160014,238900.43901975403,6314994176,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456 +ademamix,2,1337,268435456,0.5000076295109483,4096,4.015315055847168,55.440609846288844,0.31633567810058594,64,1123.6289774160014,238900.43901975403,6314994176,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_ademamix_seed2_lrscale1_tokens268435456 +muon,1,1337,16777216,0.03125047684443427,256,5.1458635330200195,171.71911191532246,0.21457481384277344,64,1163.3150507089977,230750.43672511456,6117368832,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456 +muon,1,1337,33554432,0.06250095368886854,512,4.69781494140625,109.70684704341419,0.2564544677734375,64,1163.3150507089977,230750.43672511456,6117368832,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456 +muon,1,1337,67108864,0.1250019073777371,1024,4.402804374694824,81.67936610685706,0.28281402587890625,64,1163.3150507089977,230750.43672511456,6117368832,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456 +muon,1,1337,134217728,0.2500038147554742,2048,4.192287445068359,66.17380072145147,0.3006629943847656,64,1163.3150507089977,230750.43672511456,6117368832,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456 +muon,1,1337,268435456,0.5000076295109483,4096,4.0437235832214355,57.03817971687212,0.3136253356933594,64,1163.3150507089977,230750.43672511456,6117368832,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_muon_seed1_lrscale3_tokens268435456 +muon,2,1337,16777216,0.03125047684443427,256,5.148187637329102,172.1186689007199,0.21408653259277344,64,1180.8367941000026,227326.46657118542,6117368832,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456 +muon,2,1337,33554432,0.06250095368886854,512,4.698286056518555,109.75854373879675,0.2569561004638672,64,1180.8367941000026,227326.46657118542,6117368832,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456 +muon,2,1337,67108864,0.1250019073777371,1024,4.398398399353027,81.32028071698167,0.28228759765625,64,1180.8367941000026,227326.46657118542,6117368832,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456 +muon,2,1337,134217728,0.2500038147554742,2048,4.190174102783203,66.03410059327533,0.3003520965576172,64,1180.8367941000026,227326.46657118542,6117368832,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456 +muon,2,1337,268435456,0.5000076295109483,4096,4.04427433013916,57.06960194946216,0.3135662078857422,64,1180.8367941000026,227326.46657118542,6117368832,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_muon_seed2_lrscale3_tokens268435456 +soap,1,1337,16777216,0.03125047684443427,256,5.308291912078857,202.0041701392595,0.20886802673339844,64,1276.103207307998,210355.6001291445,6559245312,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456 +soap,1,1337,33554432,0.06250095368886854,512,4.7151780128479,111.62832669491085,0.25631141662597656,64,1276.103207307998,210355.6001291445,6559245312,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456 +soap,1,1337,67108864,0.1250019073777371,1024,4.377788543701172,79.66143558302677,0.2838420867919922,64,1276.103207307998,210355.6001291445,6559245312,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456 +soap,1,1337,134217728,0.2500038147554742,2048,4.175849437713623,65.09492757975597,0.2997779846191406,64,1276.103207307998,210355.6001291445,6559245312,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456 +soap,1,1337,268435456,0.5000076295109483,4096,4.037361145019531,56.67643009174277,0.3121681213378906,64,1276.103207307998,210355.6001291445,6559245312,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_soap_seed1_lrscale1_tokens268435456 +soap,2,1337,16777216,0.03125047684443427,256,5.2917256355285645,198.68528213612078,0.21008682250976562,64,1281.7409867240024,209430.34418060805,6559245312,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456 +soap,2,1337,33554432,0.06250095368886854,512,4.710265636444092,111.08131137740004,0.25705718994140625,64,1281.7409867240024,209430.34418060805,6559245312,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456 +soap,2,1337,67108864,0.1250019073777371,1024,4.380484580993652,79.87649541386698,0.28352928161621094,64,1281.7409867240024,209430.34418060805,6559245312,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456 +soap,2,1337,134217728,0.2500038147554742,2048,4.179245948791504,65.31639897553266,0.3002758026123047,64,1281.7409867240024,209430.34418060805,6559245312,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456 +soap,2,1337,268435456,0.5000076295109483,4096,4.037126541137695,56.66313514976101,0.312255859375,64,1281.7409867240024,209430.34418060805,6559245312,/root/autodl-tmp/llm-optimizer-results/formal/sub1_confirm_token-bin_soap_seed2_lrscale1_tokens268435456 +adafactor,0,1337,16777216,0.03125047684443427,256,5.914063930511475,370.2061276976988,0.18108749389648438,64,1182.1543548539998,227073.10166205216,6040051712,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456 +adafactor,0,1337,33554432,0.06250095368886854,512,5.436375141143799,229.60753557404612,0.2048778533935547,64,1182.1543548539998,227073.10166205216,6040051712,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456 +adafactor,0,1337,67108864,0.1250019073777371,1024,5.0308027267456055,153.05530746366406,0.2279491424560547,64,1182.1543548539998,227073.10166205216,6040051712,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456 +adafactor,0,1337,134217728,0.2500038147554742,2048,4.611979007720947,100.6828931008601,0.2649803161621094,64,1182.1543548539998,227073.10166205216,6040051712,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456 +adafactor,0,1337,268435456,0.5000076295109483,4096,4.321353912353516,75.29027719728664,0.2892436981201172,64,1182.1543548539998,227073.10166205216,6040051712,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adafactor_seed0_lrscale0p3_tokens268435456 +adamw,0,1337,16777216,0.03125047684443427,256,5.626274585723877,277.6248664569638,0.1855449676513672,64,1098.9983692469998,244254.64451227876,6175430656,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456 +adamw,0,1337,33554432,0.06250095368886854,512,5.043008327484131,154.93488552851235,0.22411346435546875,64,1098.9983692469998,244254.64451227876,6175430656,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456 +adamw,0,1337,67108864,0.1250019073777371,1024,4.544707298278809,94.13258255880584,0.2710132598876953,64,1098.9983692469998,244254.64451227876,6175430656,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456 +adamw,0,1337,134217728,0.2500038147554742,2048,4.271139621734619,71.60298397602084,0.29346275329589844,64,1098.9983692469998,244254.64451227876,6175430656,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456 +adamw,0,1337,268435456,0.5000076295109483,4096,4.083443641662598,59.34933373277334,0.3088035583496094,64,1098.9983692469998,244254.64451227876,6175430656,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adamw_seed0_lrscale1_tokens268435456 +ademamix,0,1337,16777216,0.03125047684443427,256,5.676942825317383,292.0540847041283,0.18198013305664062,64,1127.8803733319994,237999.93540714282,6314994176,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456 +ademamix,0,1337,33554432,0.06250095368886854,512,5.0648016929626465,158.34849776052857,0.22062301635742188,64,1127.8803733319994,237999.93540714282,6314994176,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456 +ademamix,0,1337,67108864,0.1250019073777371,1024,4.498353958129883,89.86880928696887,0.2741413116455078,64,1127.8803733319994,237999.93540714282,6314994176,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456 +ademamix,0,1337,134217728,0.2500038147554742,2048,4.205862522125244,67.07823958130747,0.2999076843261719,64,1127.8803733319994,237999.93540714282,6314994176,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456 +ademamix,0,1337,268435456,0.5000076295109483,4096,4.01993989944458,55.69760765215066,0.31641578674316406,64,1127.8803733319994,237999.93540714282,6314994176,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_ademamix_seed0_lrscale1_tokens268435456 +adopt,0,1337,16777216,0.03125047684443427,256,5.867005348205566,353.18832093529016,0.180328369140625,64,1122.9521831029995,239044.4224065225,6175400960,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456 +adopt,0,1337,33554432,0.06250095368886854,512,5.324961185455322,205.3996520755454,0.21122360229492188,64,1122.9521831029995,239044.4224065225,6175400960,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456 +adopt,0,1337,67108864,0.1250019073777371,1024,4.819103240966797,123.85357231261972,0.24735641479492188,64,1122.9521831029995,239044.4224065225,6175400960,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456 +adopt,0,1337,134217728,0.2500038147554742,2048,4.413585662841797,82.56473844410924,0.2838916778564453,64,1122.9521831029995,239044.4224065225,6175400960,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456 +adopt,0,1337,268435456,0.5000076295109483,4096,4.170770168304443,64.76513139810707,0.3033905029296875,64,1122.9521831029995,239044.4224065225,6175400960,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_adopt_seed0_lrscale0p3_tokens268435456 +d-muon,0,1337,16777216,0.03125047684443427,256,5.119787216186523,167.2991909985406,0.21628952026367188,64,1153.460835318,232721.77761110934,6252898816,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456 +d-muon,0,1337,33554432,0.06250095368886854,512,4.61951208114624,101.4442081359866,0.2632331848144531,64,1153.460835318,232721.77761110934,6252898816,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456 +d-muon,0,1337,67108864,0.1250019073777371,1024,4.364060401916504,78.57530516513165,0.28444671630859375,64,1153.460835318,232721.77761110934,6252898816,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456 +d-muon,0,1337,134217728,0.2500038147554742,2048,4.189532279968262,65.99173202746029,0.2987251281738281,64,1153.460835318,232721.77761110934,6252898816,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456 +d-muon,0,1337,268435456,0.5000076295109483,4096,4.057147979736328,57.80904496300566,0.30995750427246094,64,1153.460835318,232721.77761110934,6252898816,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_d-muon_seed0_lrscale3_tokens268435456 +lamb,0,1337,16777216,0.03125047684443427,256,6.028428077697754,415.06038790102406,0.16856956481933594,64,1158.5195744759978,231705.5852262265,6175793152,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456 +lamb,0,1337,33554432,0.06250095368886854,512,5.410538196563721,223.75116350202543,0.202606201171875,64,1158.5195744759978,231705.5852262265,6175793152,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456 +lamb,0,1337,67108864,0.1250019073777371,1024,4.8422064781188965,126.74829883936543,0.24698448181152344,64,1158.5195744759978,231705.5852262265,6175793152,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456 +lamb,0,1337,134217728,0.2500038147554742,2048,4.455793380737305,86.1241950969045,0.2796516418457031,64,1158.5195744759978,231705.5852262265,6175793152,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456 +lamb,0,1337,268435456,0.5000076295109483,4096,4.210540771484375,67.3927832843249,0.29844093322753906,64,1158.5195744759978,231705.5852262265,6175793152,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_lamb_seed0_lrscale27_tokens268435456 +lion,0,1337,16777216,0.03125047684443427,256,6.305228233337402,547.4241931523586,0.14279747009277344,64,1117.527238096001,240204.84409610435,6039584256,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456 +lion,0,1337,33554432,0.06250095368886854,512,5.674002647399902,291.19665542532005,0.18014907836914062,64,1117.527238096001,240204.84409610435,6039584256,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456 +lion,0,1337,67108864,0.1250019073777371,1024,5.055916786193848,156.94781872872173,0.2221813201904297,64,1117.527238096001,240204.84409610435,6039584256,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456 +lion,0,1337,134217728,0.2500038147554742,2048,4.4709062576293945,87.43566364260828,0.27742576599121094,64,1117.527238096001,240204.84409610435,6039584256,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456 +lion,0,1337,268435456,0.5000076295109483,4096,4.166975021362305,64.51980419359604,0.3036460876464844,64,1117.527238096001,240204.84409610435,6039584256,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_lion_seed0_lrscale0p3_tokens268435456 +mars,0,1337,16777216,0.03125047684443427,256,5.857702732086182,349.91798270233926,0.1775531768798828,64,1150.9215605589964,233235.23096536862,6324693504,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456 +mars,0,1337,33554432,0.06250095368886854,512,5.304955005645752,201.3312249790169,0.20838356018066406,64,1150.9215605589964,233235.23096536862,6324693504,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456 +mars,0,1337,67108864,0.1250019073777371,1024,4.750679016113281,115.66242516418683,0.2524585723876953,64,1150.9215605589964,233235.23096536862,6324693504,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456 +mars,0,1337,134217728,0.2500038147554742,2048,4.3549089431762695,77.85950727643227,0.28768157958984375,64,1150.9215605589964,233235.23096536862,6324693504,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456 +mars,0,1337,268435456,0.5000076295109483,4096,4.1218061447143555,61.670356648930785,0.30554962158203125,64,1150.9215605589964,233235.23096536862,6324693504,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_mars_seed0_lrscale0p3_tokens268435456 +muon-pytorch,0,1337,16777216,0.03125047684443427,256,5.278108596801758,195.99811590753652,0.2044086456298828,64,1148.5943301670006,233707.80174491255,6117270528,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456 +muon-pytorch,0,1337,33554432,0.06250095368886854,512,4.948966979980469,141.0287331811823,0.23703956604003906,64,1148.5943301670006,233707.80174491255,6117270528,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456 +muon-pytorch,0,1337,67108864,0.1250019073777371,1024,4.761952877044678,116.97376441736354,0.2509288787841797,64,1148.5943301670006,233707.80174491255,6117270528,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456 +muon-pytorch,0,1337,134217728,0.2500038147554742,2048,4.571454048156738,96.68429436191391,0.26610374450683594,64,1148.5943301670006,233707.80174491255,6117270528,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456 +muon-pytorch,0,1337,268435456,0.5000076295109483,4096,4.4571852684021,86.24415368608321,0.27433204650878906,64,1148.5943301670006,233707.80174491255,6117270528,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_muon-pytorch_seed0_lrscale3_tokens268435456 +muon,0,1337,16777216,0.03125047684443427,256,5.167823791503906,175.53181619810817,0.2129058837890625,64,1175.974133417001,228266.4629875942,6117368832,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456 +muon,0,1337,33554432,0.06250095368886854,512,4.705371856689453,110.53903224899439,0.2558574676513672,64,1175.974133417001,228266.4629875942,6117368832,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456 +muon,0,1337,67108864,0.1250019073777371,1024,4.408752918243408,82.16668703325017,0.2809886932373047,64,1175.974133417001,228266.4629875942,6117368832,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456 +muon,0,1337,134217728,0.2500038147554742,2048,4.191479682922363,66.12036964881675,0.300384521484375,64,1175.974133417001,228266.4629875942,6117368832,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456 +muon,0,1337,268435456,0.5000076295109483,4096,4.043658256530762,57.03445372556012,0.3135967254638672,64,1175.974133417001,228266.4629875942,6117368832,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_muon_seed0_lrscale3_tokens268435456 +prodigy,0,1337,16777216,0.03125047684443427,256,5.787520885467529,326.2020558131375,0.1793346405029297,64,1147.7304891460017,233883.70226161394,6451916800,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456 +prodigy,0,1337,33554432,0.06250095368886854,512,5.105589866638184,164.9407687779071,0.2185993194580078,64,1147.7304891460017,233883.70226161394,6451916800,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456 +prodigy,0,1337,67108864,0.1250019073777371,1024,4.548550128936768,94.49501382416213,0.2702503204345703,64,1147.7304891460017,233883.70226161394,6451916800,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456 +prodigy,0,1337,134217728,0.2500038147554742,2048,4.271369934082031,71.61947691543894,0.29283905029296875,64,1147.7304891460017,233883.70226161394,6451916800,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456 +prodigy,0,1337,268435456,0.5000076295109483,4096,4.091464042663574,59.82725285657247,0.3072776794433594,64,1147.7304891460017,233883.70226161394,6451916800,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_prodigy_seed0_lrscale0p3_tokens268435456 +scion-light,0,1337,16777216,0.03125047684443427,256,6.452136039733887,634.0524626588718,0.1423053741455078,64,1171.1609263130017,229204.5866361653,5909968384,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456 +scion-light,0,1337,33554432,0.06250095368886854,512,6.336269378662109,564.6833415248443,0.14540672302246094,64,1171.1609263130017,229204.5866361653,5909968384,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456 +scion-light,0,1337,67108864,0.1250019073777371,1024,6.41378116607666,610.1939465694576,0.14218521118164062,64,1171.1609263130017,229204.5866361653,5909968384,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456 +scion-light,0,1337,134217728,0.2500038147554742,2048,6.293587684631348,541.088824551898,0.146240234375,64,1171.1609263130017,229204.5866361653,5909968384,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456 +scion-light,0,1337,268435456,0.5000076295109483,4096,6.240045070648193,512.8794735730139,0.14815711975097656,64,1171.1609263130017,229204.5866361653,5909968384,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_scion-light_seed0_lrscale0p3_tokens268435456 +scion,0,1337,16777216,0.03125047684443427,256,6.25499153137207,520.6027755209898,0.14007186889648438,64,1175.894496335001,228281.92226143845,6039584256,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456 +scion,0,1337,33554432,0.06250095368886854,512,6.163638591766357,475.1518540963069,0.14553451538085938,64,1175.894496335001,228281.92226143845,6039584256,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456 +scion,0,1337,67108864,0.1250019073777371,1024,6.05860710144043,427.77742535588936,0.14849853515625,64,1175.894496335001,228281.92226143845,6039584256,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456 +scion,0,1337,134217728,0.2500038147554742,2048,6.074914932250977,434.810735647104,0.14995193481445312,64,1175.894496335001,228281.92226143845,6039584256,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456 +scion,0,1337,268435456,0.5000076295109483,4096,6.072164535522461,433.6164775202488,0.15175628662109375,64,1175.894496335001,228281.92226143845,6039584256,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_scion_seed0_lrscale0p3_tokens268435456 +sf-adamw,0,1337,16777216,0.03125047684443427,256,6.022677421569824,412.680369830492,0.1676197052001953,64,1116.8707595270018,240346.03261856656,6175400960,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456 +sf-adamw,0,1337,33554432,0.06250095368886854,512,5.567577362060547,261.79810550128695,0.19131088256835938,64,1116.8707595270018,240346.03261856656,6175400960,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456 +sf-adamw,0,1337,67108864,0.1250019073777371,1024,5.125164031982422,168.20114998532256,0.2177581787109375,64,1116.8707595270018,240346.03261856656,6175400960,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456 +sf-adamw,0,1337,134217728,0.2500038147554742,2048,4.690937042236328,108.95488384486792,0.2512531280517578,64,1116.8707595270018,240346.03261856656,6175400960,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456 +sf-adamw,0,1337,268435456,0.5000076295109483,4096,4.304592132568359,74.0387968129434,0.29078102111816406,64,1116.8707595270018,240346.03261856656,6175400960,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sf-adamw_seed0_lrscale1_tokens268435456 +sf-sgd,0,1337,16777216,0.03125047684443427,256,7.614336967468262,2027.0399306956288,0.05343437194824219,64,1108.248907474,242215.85438945956,6039977472,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456 +sf-sgd,0,1337,33554432,0.06250095368886854,512,7.473572731018066,1760.877782745687,0.059734344482421875,64,1108.248907474,242215.85438945956,6039977472,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456 +sf-sgd,0,1337,67108864,0.1250019073777371,1024,7.332432270050049,1529.0888863205523,0.08491325378417969,64,1108.248907474,242215.85438945956,6039977472,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456 +sf-sgd,0,1337,134217728,0.2500038147554742,2048,7.18770170211792,1323.052528344451,0.08907699584960938,64,1108.248907474,242215.85438945956,6039977472,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456 +sf-sgd,0,1337,268435456,0.5000076295109483,4096,6.992140293121338,1088.042609550867,0.10650634765625,64,1108.248907474,242215.85438945956,6039977472,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sf-sgd_seed0_lrscale729_tokens268435456 +sgd,0,1337,16777216,0.03125047684443427,256,7.567975044250488,1935.2077298538604,0.051807403564453125,64,1095.110613765004,245121.77366002832,6039715328,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456 +sgd,0,1337,33554432,0.06250095368886854,512,7.446283340454102,1713.474281203508,0.052227020263671875,64,1095.110613765004,245121.77366002832,6039715328,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456 +sgd,0,1337,67108864,0.1250019073777371,1024,7.361580848693848,1574.3155681500446,0.049957275390625,64,1095.110613765004,245121.77366002832,6039715328,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456 +sgd,0,1337,134217728,0.2500038147554742,2048,7.295380592346191,1473.4703577673076,0.05741691589355469,64,1095.110613765004,245121.77366002832,6039715328,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456 +sgd,0,1337,268435456,0.5000076295109483,4096,6.9730939865112305,1067.5155332391428,0.10369110107421875,64,1095.110613765004,245121.77366002832,6039715328,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sgd_seed0_lrscale9_tokens268435456 +signsgd,0,1337,16777216,0.03125047684443427,256,6.4919915199279785,659.8332509822862,0.15250015258789062,64,1112.2371516669982,241347.32021644345,5909968384,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456 +signsgd,0,1337,33554432,0.06250095368886854,512,6.202166557312012,493.8157072855017,0.1693267822265625,64,1112.2371516669982,241347.32021644345,5909968384,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456 +signsgd,0,1337,67108864,0.1250019073777371,1024,5.864711761474609,352.3791817033458,0.19159698486328125,64,1112.2371516669982,241347.32021644345,5909968384,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456 +signsgd,0,1337,134217728,0.2500038147554742,2048,5.575797080993652,263.95887920271304,0.21255874633789062,64,1112.2371516669982,241347.32021644345,5909968384,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456 +signsgd,0,1337,268435456,0.5000076295109483,4096,5.291903495788574,198.7206234711055,0.2403564453125,64,1112.2371516669982,241347.32021644345,5909968384,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_signsgd_seed0_lrscale0p3_tokens268435456 +signum,0,1337,16777216,0.03125047684443427,256,6.17771053314209,481.8854246932734,0.15179061889648438,64,1119.0858592749996,239870.29571967432,6040567296,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456 +signum,0,1337,33554432,0.06250095368886854,512,5.435019493103027,229.2964796663352,0.19580841064453125,64,1119.0858592749996,239870.29571967432,6040567296,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456 +signum,0,1337,67108864,0.1250019073777371,1024,4.796474933624268,121.08244863877887,0.25129127502441406,64,1119.0858592749996,239870.29571967432,6040567296,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456 +signum,0,1337,134217728,0.2500038147554742,2048,4.4577717781066895,86.29475152175984,0.27893638610839844,64,1119.0858592749996,239870.29571967432,6040567296,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456 +signum,0,1337,268435456,0.5000076295109483,4096,4.239973068237305,69.4057846405103,0.29747962951660156,64,1119.0858592749996,239870.29571967432,6040567296,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_signum_seed0_lrscale1_tokens268435456 +soap,0,1337,16777216,0.03125047684443427,256,5.292172431945801,198.77407378302857,0.20963287353515625,64,1284.3550443230042,209004.0890067862,6559245312,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456 +soap,0,1337,33554432,0.06250095368886854,512,4.70062780380249,110.0158715181136,0.25766944885253906,64,1284.3550443230042,209004.0890067862,6559245312,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456 +soap,0,1337,67108864,0.1250019073777371,1024,4.373493194580078,79.31999596153322,0.2840461730957031,64,1284.3550443230042,209004.0890067862,6559245312,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456 +soap,0,1337,134217728,0.2500038147554742,2048,4.17411470413208,64.98210318721175,0.3007545471191406,64,1284.3550443230042,209004.0890067862,6559245312,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456 +soap,0,1337,268435456,0.5000076295109483,4096,4.038823127746582,56.75935059719494,0.31163597106933594,64,1284.3550443230042,209004.0890067862,6559245312,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_soap_seed0_lrscale1_tokens268435456 +sophiag,0,1337,16777216,0.03125047684443427,256,6.090432167053223,441.6104109948902,0.15760040283203125,64,1145.1167670799987,234417.54039153538,14279713792,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456 +sophiag,0,1337,33554432,0.06250095368886854,512,5.460256576538086,235.15688888849053,0.19549942016601562,64,1145.1167670799987,234417.54039153538,14279713792,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456 +sophiag,0,1337,67108864,0.1250019073777371,1024,4.812508583068848,123.03948817490895,0.24817276000976562,64,1145.1167670799987,234417.54039153538,14279713792,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456 +sophiag,0,1337,134217728,0.2500038147554742,2048,4.430809497833252,83.99913636632618,0.28174591064453125,64,1145.1167670799987,234417.54039153538,14279713792,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456 +sophiag,0,1337,268435456,0.5000076295109483,4096,4.209933280944824,67.35185526652654,0.29942893981933594,64,1145.1167670799987,234417.54039153538,14279713792,/root/autodl-tmp/llm-optimizer-results/formal/sub1_main_token-bin_sophiag_seed0_lrscale0p3_tokens268435456 +adafactor,0,1337,16777216,0.03125047684443427,256,5.9108781814575195,369.0286212845167,0.1810016632080078,64,74.08217012899999,226467.66382228915,6040051712,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p1_tokens16777216 +adafactor,0,1337,16777216,0.03125047684443427,256,5.906637191772461,367.46688974311246,0.1814289093017578,64,74.959563125,223816.8860726001,6040051712,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale0p3_tokens16777216 +adafactor,0,1337,16777216,0.03125047684443427,256,5.910003662109375,368.7060399045575,0.18126869201660156,64,74.32661689599998,225722.85273625707,6040051712,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale1_tokens16777216 +adafactor,0,1337,16777216,0.03125047684443427,256,5.909377574920654,368.4752701803194,0.18115806579589844,64,74.47961303800007,225259.1724857665,6040051712,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adafactor_seed0_lrscale3_tokens16777216 +adamw,0,1337,16777216,0.03125047684443427,256,5.825977802276611,338.9911101928887,0.18766212463378906,64,70.02755577000005,239580.2026148597,6175430656,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adamw_seed0_lrscale0p3_tokens16777216 +adamw,0,1337,16777216,0.03125047684443427,256,5.633615016937256,279.67024912779885,0.1858081817626953,64,70.83568091299999,236846.9644642181,6175430656,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adamw_seed0_lrscale1_tokens16777216 +adamw,0,1337,16777216,0.03125047684443427,256,6.316389083862305,553.5681306207249,0.14019012451171875,64,69.917800776,239956.28886769776,6175430656,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adamw_seed0_lrscale3_tokens16777216 +ademamix,0,1337,16777216,0.03125047684443427,256,5.787484645843506,326.1902345954303,0.18692588806152344,64,72.67416188899998,230855.30763498787,6314994176,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_ademamix_seed0_lrscale0p3_tokens16777216 +ademamix,0,1337,16777216,0.03125047684443427,256,5.664802074432373,288.5297684001515,0.1833667755126953,64,71.56953362500002,234418.40613223636,6314994176,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_ademamix_seed0_lrscale1_tokens16777216 +ademamix,0,1337,16777216,0.03125047684443427,256,6.46177864074707,640.1959454203139,0.1282176971435547,64,71.97123666199998,233110.01419624328,6314994176,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_ademamix_seed0_lrscale3_tokens16777216 +adopt,0,1337,16777216,0.03125047684443427,256,6.21639347076416,500.8913890653622,0.1710338592529297,64,70.27018207499995,238752.98888643176,6175400960,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p1_tokens16777216 +adopt,0,1337,16777216,0.03125047684443427,256,5.876038551330566,356.3931939656842,0.179473876953125,64,69.73402755099995,240588.65648811104,6175400960,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale0p3_tokens16777216 +adopt,0,1337,16777216,0.03125047684443427,256,6.103500843048096,447.4195466420294,0.1589946746826172,64,69.85458485299993,240173.44080285513,6175400960,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale1_tokens16777216 +adopt,0,1337,16777216,0.03125047684443427,256,6.369908332824707,584.0017901147743,0.13656234741210938,64,70.62556766500003,237551.59150832988,6175400960,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_adopt_seed0_lrscale3_tokens16777216 +d-muon,0,1337,16777216,0.03125047684443427,256,6.030533790588379,415.9353061617444,0.1616668701171875,64,73.275967925,228959.3228870337,6252898816,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale0p3_tokens16777216 +d-muon,0,1337,16777216,0.03125047684443427,256,5.397306442260742,220.81004604229818,0.1990680694580078,64,72.49320627200002,231431.56252532976,6386240000,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale1_tokens16777216 +d-muon,0,1337,16777216,0.03125047684443427,256,5.1279730796813965,168.6742989582807,0.2153148651123047,64,72.539968561,231282.37208831674,6252898816,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale3_tokens16777216 +d-muon,0,1337,16777216,0.03125047684443427,256,5.191718101501465,179.77653533712166,0.21042633056640625,64,72.07763624499998,232765.90179750565,6252898816,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_d-muon_seed0_lrscale9_tokens16777216 +lamb,0,1337,16777216,0.03125047684443427,256,9.87247085571289,19389.058962766347,0.0887298583984375,64,72.16825172200001,232473.63764093482,6175793152,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale0p3_tokens16777216 +lamb,0,1337,16777216,0.03125047684443427,256,9.244894027709961,10351.511318109584,0.1311664581298828,64,72.92484238200004,230061.73824986015,6175793152,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale1_tokens16777216 +lamb,0,1337,16777216,0.03125047684443427,256,5.966597557067871,390.17429268796366,0.17175865173339844,64,73.24245941699994,229064.07203614363,6175793152,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale27_tokens16777216 +lamb,0,1337,16777216,0.03125047684443427,256,7.944293975830078,2819.4260985800843,0.15624427795410156,64,75.17631810899998,223171.557506638,6175793152,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale3_tokens16777216 +lamb,0,1337,16777216,0.03125047684443427,256,6.549368381500244,698.7995791757087,0.12223625183105469,64,73.31413933900005,228840.1139434126,6175793152,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale81_tokens16777216 +lamb,0,1337,16777216,0.03125047684443427,256,6.3077802658081055,548.8230206950632,0.1752777099609375,64,72.42151405900002,231660.6635195726,6175793152,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lamb_seed0_lrscale9_tokens16777216 +lion,0,1337,16777216,0.03125047684443427,256,6.423236846923828,615.9911067382695,0.1471996307373047,64,70.83748797199996,236840.9225159361,6039584256,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale0p1_tokens16777216 +lion,0,1337,16777216,0.03125047684443427,256,6.276636123657227,531.9938353895492,0.14625167846679688,64,70.65828603599996,237441.59307023257,6039584256,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale0p3_tokens16777216 +lion,0,1337,16777216,0.03125047684443427,256,6.849046230316162,942.9767465226221,0.09955406188964844,64,71.19586444,235648.74353255343,6039584256,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale1_tokens16777216 +lion,0,1337,16777216,0.03125047684443427,256,7.386219024658203,1613.585590064328,0.07188224792480469,64,69.997784646,239682.09972426217,6039584256,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_lion_seed0_lrscale3_tokens16777216 +mars,0,1337,16777216,0.03125047684443427,256,6.105465412139893,448.2993966426686,0.17698097229003906,64,71.78387870099995,233718.43795013955,6324693504,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale0p1_tokens16777216 +mars,0,1337,16777216,0.03125047684443427,256,5.844259262084961,345.24535249628593,0.17828369140625,64,72.59430473599997,231109.25934221494,6324693504,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale0p3_tokens16777216 +mars,0,1337,16777216,0.03125047684443427,256,6.1865925788879395,486.18461472526104,0.1516857147216797,64,71.45638488799999,234789.599645944,6324693504,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale1_tokens16777216 +mars,0,1337,16777216,0.03125047684443427,256,6.590240478515625,727.9526796275507,0.12089157104492188,64,71.51491461100001,234597.44154430446,6324693504,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_mars_seed0_lrscale3_tokens16777216 +muon-pytorch,0,1337,16777216,0.03125047684443427,256,5.793531894683838,328.1687630800648,0.192840576171875,64,71.583995558,234371.04717641082,6117270528,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale0p3_tokens16777216 +muon-pytorch,0,1337,16777216,0.03125047684443427,256,5.356301784515381,211.9389330623356,0.20544815063476562,64,71.78204306099995,233724.41469439398,6117270528,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale1_tokens16777216 +muon-pytorch,0,1337,16777216,0.03125047684443427,256,5.283370018005371,197.03206148135158,0.20551109313964844,64,71.56041207699997,234448.2866021997,6117270528,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale3_tokens16777216 +muon-pytorch,0,1337,16777216,0.03125047684443427,256,5.898062705993652,364.32952211229554,0.16612625122070312,64,71.63904080699997,234190.96362832192,6117270528,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon-pytorch_seed0_lrscale9_tokens16777216 +muon,0,1337,16777216,0.03125047684443427,256,5.847861289978027,346.4911774490298,0.18353843688964844,64,75.548639173,222071.7167596043,6117368832,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale0p3_tokens16777216 +muon,0,1337,16777216,0.03125047684443427,256,5.261423110961914,192.75492663883713,0.21074295043945312,64,74.21483872099998,226062.82367696753,6117368832,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale1_tokens16777216 +muon,0,1337,16777216,0.03125047684443427,256,5.157479286193848,173.72538702515928,0.2135181427001953,64,74.58441240600004,224942.65837576453,6117368832,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale3_tokens16777216 +muon,0,1337,16777216,0.03125047684443427,256,5.522243499755859,250.19479268436115,0.18825912475585938,64,73.899425389,227027.6921868638,6117368832,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_muon_seed0_lrscale9_tokens16777216 +prodigy,0,1337,16777216,0.03125047684443427,256,6.069522857666016,432.47251489850487,0.17177772521972656,64,72.63071371999999,230993.4067931393,6451916800,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p1_tokens16777216 +prodigy,0,1337,16777216,0.03125047684443427,256,5.899776458740234,364.95442772504396,0.1726512908935547,64,72.33771752700002,231929.0208975409,6451916800,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale0p3_tokens16777216 +prodigy,0,1337,16777216,0.03125047684443427,256,6.313230991363525,551.822670047611,0.1423320770263672,64,72.822075675,230386.4019871609,6451916800,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale1_tokens16777216 +prodigy,0,1337,16777216,0.03125047684443427,256,6.386815071105957,593.9592860424405,0.13541412353515625,64,71.42516241400007,234892.23451470223,6451916800,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_prodigy_seed0_lrscale3_tokens16777216 +scion-light,0,1337,16777216,0.03125047684443427,256,6.558917999267578,705.5048087754254,0.1382732391357422,64,73.74715544899999,227496.44915595857,5909968384,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p1_tokens16777216 +scion-light,0,1337,16777216,0.03125047684443427,256,6.453986167907715,635.2266260353618,0.14255905151367188,64,72.206245666,232351.31317594516,5909968384,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale0p3_tokens16777216 +scion-light,0,1337,16777216,0.03125047684443427,256,6.479294776916504,651.5084839119285,0.11521148681640625,64,73.16113002599994,229318.71055077645,5909968384,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale1_tokens16777216 +scion-light,0,1337,16777216,0.03125047684443427,256,6.810400009155273,907.2294800769788,0.10309219360351562,64,73.39754613599999,228580.06681739897,5909968384,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion-light_seed0_lrscale3_tokens16777216 +scion,0,1337,16777216,0.03125047684443427,256,6.358173370361328,577.1886100048764,0.1439380645751953,64,73.12712026200005,229425.36147862166,6039584256,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale0p1_tokens16777216 +scion,0,1337,16777216,0.03125047684443427,256,6.25476598739624,520.4853700206932,0.14045143127441406,64,72.59823264799998,231096.7552246907,6039584256,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale0p3_tokens16777216 +scion,0,1337,16777216,0.03125047684443427,256,6.319926738739014,555.52993033717,0.12175750732421875,64,73.90401923500002,227013.58023102646,6039584256,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale1_tokens16777216 +scion,0,1337,16777216,0.03125047684443427,256,6.581727027893066,721.7816004465782,0.113983154296875,64,73.70046004400001,227640.58718200418,6039584256,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_scion_seed0_lrscale3_tokens16777216 +sf-adamw,0,1337,16777216,0.03125047684443427,256,6.211974143981934,498.6826719365267,0.17000961303710938,64,69.91316692800001,239972.19318183648,6175400960,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale0p3_tokens16777216 +sf-adamw,0,1337,16777216,0.03125047684443427,256,6.035920143127441,418.1817234095075,0.16771888732910156,64,69.51470453899998,241347.7279557082,6175400960,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale1_tokens16777216 +sf-adamw,0,1337,16777216,0.03125047684443427,256,6.4182233810424805,612.9105869064393,0.1320209503173828,64,69.997612869,239682.6879139212,6175400960,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-adamw_seed0_lrscale3_tokens16777216 +sf-sgd,0,1337,16777216,0.03125047684443427,256,10.325323104858398,30494.943424849957,0.0346832275390625,64,69.64091573300004,240910.33013297882,6039977472,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale0p3_tokens16777216 +sf-sgd,0,1337,16777216,0.03125047684443427,256,9.711913108825684,16513.055279307664,0.03781890869140625,64,68.88036811200004,243570.35915836148,6039977472,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale1_tokens16777216 +sf-sgd,0,1337,16777216,0.03125047684443427,256,7.5896501541137695,1977.611433827244,0.05286407470703125,64,69.21735874999996,242384.51600842123,6039977472,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale243_tokens16777216 +sf-sgd,0,1337,16777216,0.03125047684443427,256,7.711722373962402,2234.3758060934124,0.08671760559082031,64,70.50132499499999,237970.22256233983,6039977472,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale27_tokens16777216 +sf-sgd,0,1337,16777216,0.03125047684443427,256,8.906181335449219,7377.391093153199,0.03958892822265625,64,69.46517621699999,241519.80767442673,6039977472,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale3_tokens16777216 +sf-sgd,0,1337,16777216,0.03125047684443427,256,7.577207565307617,1953.1572962701177,0.05061912536621094,64,69.78736949199998,240404.7626687411,6039977472,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale729_tokens16777216 +sf-sgd,0,1337,16777216,0.03125047684443427,256,7.60025691986084,1998.6991192268076,0.08129692077636719,64,70.46290074299996,238099.99053532732,6039977472,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale81_tokens16777216 +sf-sgd,0,1337,16777216,0.03125047684443427,256,8.234554290771484,3768.938786317613,0.06800079345703125,64,70.36729345900002,238423.494428919,6039977472,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sf-sgd_seed0_lrscale9_tokens16777216 +sgd,0,1337,16777216,0.03125047684443427,256,8.42439079284668,4556.842033847346,0.05838584899902344,64,71.36861597700005,235078.3431950928,6039715328,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale0p3_tokens16777216 +sgd,0,1337,16777216,0.03125047684443427,256,7.8472442626953125,2558.6600544809194,0.07884788513183594,64,70.252648892,238812.57524953628,6039715328,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale1_tokens16777216 +sgd,0,1337,16777216,0.03125047684443427,256,7.623948574066162,2046.616960538731,0.06693267822265625,64,70.37890287699994,238384.1650575495,6039715328,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale27_tokens16777216 +sgd,0,1337,16777216,0.03125047684443427,256,7.7101545333862305,2230.875408146757,0.053325653076171875,64,70.45152964899997,238138.42060756657,6039715328,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale3_tokens16777216 +sgd,0,1337,16777216,0.03125047684443427,256,7.59018611907959,1978.6716476519782,0.047283172607421875,64,70.11675072299994,239275.43457168047,6039715328,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sgd_seed0_lrscale9_tokens16777216 +signsgd,0,1337,16777216,0.03125047684443427,256,6.661827087402344,781.9748714287975,0.14697837829589844,64,70.893569206,236653.56657737694,5909968384,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p1_tokens16777216 +signsgd,0,1337,16777216,0.03125047684443427,256,6.475642204284668,649.1331441433161,0.1517047882080078,64,68.95654871499998,243301.27178117435,5909968384,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale0p3_tokens16777216 +signsgd,0,1337,16777216,0.03125047684443427,256,6.683113098144531,798.7984033511141,0.13363075256347656,64,69.328829448,241994.79687715962,5909968384,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale1_tokens16777216 +signsgd,0,1337,16777216,0.03125047684443427,256,6.854705810546875,948.3287261686504,0.11468124389648438,64,70.11524923599995,239280.5585491082,5909968384,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signsgd_seed0_lrscale3_tokens16777216 +signum,0,1337,16777216,0.03125047684443427,256,6.296080112457275,542.4391305573229,0.14620208740234375,64,70.628779494,237540.7888993076,6040567296,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signum_seed0_lrscale0p3_tokens16777216 +signum,0,1337,16777216,0.03125047684443427,256,6.2198805809021,502.641100274047,0.14912796020507812,64,70.13492149399995,239213.44235674792,6040567296,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signum_seed0_lrscale1_tokens16777216 +signum,0,1337,16777216,0.03125047684443427,256,6.384660720825195,592.6810679067656,0.139007568359375,64,69.51772536999995,241337.24040459082,6040567296,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_signum_seed0_lrscale3_tokens16777216 +soap,0,1337,16777216,0.03125047684443427,256,6.225244998931885,505.3447206276595,0.18319320678710938,64,81.71295656799995,205318.92009118924,6556407296,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_soap_seed0_lrscale0p3_tokens16777216 +soap,0,1337,16777216,0.03125047684443427,256,5.291918754577637,198.723655728276,0.2096538543701172,64,83.30477875699998,201395.60119280955,6556407296,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_soap_seed0_lrscale1_tokens16777216 +soap,0,1337,16777216,0.03125047684443427,256,5.5883636474609375,267.29686332990445,0.18512725830078125,64,82.14935480000007,204228.21385323876,6556407296,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_soap_seed0_lrscale3_tokens16777216 +sophiag,0,1337,16777216,0.03125047684443427,256,6.111329078674316,450.935794965251,0.17660903930664062,64,72.79379595199997,230475.90499419553,14279713792,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p1_tokens16777216 +sophiag,0,1337,16777216,0.03125047684443427,256,6.050726890563965,424.4196984680204,0.160858154296875,64,72.48078954399999,231471.209206617,14279713792,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale0p3_tokens16777216 +sophiag,0,1337,16777216,0.03125047684443427,256,6.201114654541016,493.29653463131854,0.14842605590820312,64,72.68963629099997,230806.16241956988,14279713792,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale1_tokens16777216 +sophiag,0,1337,16777216,0.03125047684443427,256,6.950982093811035,1044.1698201716288,0.09879302978515625,64,71.69533521800004,234007.0793864963,14279713792,/root/autodl-tmp/llm-optimizer-results/formal/sub1_tune_token-bin_sophiag_seed0_lrscale3_tokens16777216 diff --git a/scripts/sub_one_pass/README.md b/scripts/sub_one_pass/README.md new file mode 100644 index 0000000..f5a5f4e --- /dev/null +++ b/scripts/sub_one_pass/README.md @@ -0,0 +1,139 @@ +# Sub-one-pass optimizer benchmark + +This is a new experiment and is unrelated to Datablations. It compares all 20 +optimizers exposed by this repository on the same nested prefix of a large, +fixed tokenized corpus. + +The estimand is early **sample efficiency**: validation quality after the same +number of observed training tokens. It is not an equal-FLOP or equal-wall-clock +comparison, so throughput, elapsed time and peak memory are reported separately. + +## Fixed protocol + +- Model: approximately 30M-parameter Llama (`8 × 384`, 6 heads). +- Sequence length: 512. +- Micro batch: 16; gradient accumulation: 8. +- Tokens per optimizer step: 65,536. +- Main checkpoints: 16M, 32M, 64M, 128M and 256M tokens. +- Data order: deterministic, fixed-boundary, no-replacement affine permutation. +- Schedule: 16-step linear warmup followed by constant LR. +- Optimizer-internal horizon: fixed at 256M tokens in every stage. +- Validation: the same 64 batches at every checkpoint. + +With the default 1,073,741,824-token training file, the main trajectory exposes +about 25% of the unique corpus and never begins a second pass. + +## Optimizers + +`adamw`, `sgd`, `muon`, `soap`, `ademamix`, `lion`, `sf-adamw`, `sf-sgd`, +`signsgd`, `signum`, `prodigy`, `sophiag`, `adopt`, `mars`, `adafactor`, +`lamb`, `scion`, `scion-light`, `d-muon`, `muon-pytorch`. + +`muon-pytorch` uses `torch.optim.Muon`, so all formal runs need one common +PyTorch release that provides this class. Do not run only this optimizer under a +different PyTorch version and mix it into the same ranking. + +## Prepare the fixed corpus + +```bash +python scripts/sub_one_pass/prepare_fineweb_stream.py \ + --output-dir /path/to/fineweb-1b +``` + +The command streams a bounded FineWeb sample and writes uint16 `train.bin`, +`val.bin` and `manifest.json`. It does not materialize the full upstream corpus. + +The commands below abbreviate the repeated data arguments as: + +```bash +DATA_ARGS="--dataset token-bin \ + --train-data-path /path/to/fineweb-1b/train.bin \ + --val-data-path /path/to/fineweb-1b/val.bin" +``` + +## S0: compatibility smoke + +This runs 32 steps per optimizer. Run it before any expensive queue: + +```bash +python scripts/sub_one_pass/run_all.py \ + --stage smoke \ + $DATA_ARGS \ + --results-dir ./exps/sub_one_pass +``` + +## S1: equal-budget LR screening + +Each optimizer is run at `0.3×`, `1×` and `3×` its registry LR for 16M tokens: + +```bash +python scripts/sub_one_pass/run_all.py \ + --stage tune \ + $DATA_ARGS \ + --results-dir ./exps/sub_one_pass +``` + +Select the best scale: + +```bash +python scripts/sub_one_pass/select_lr.py \ + ./exps/sub_one_pass/manifest_tune.json +``` + +Exit status 2 means at least one optimum lies on the grid boundary. Expand that +optimizer's LR grid before starting the main experiment. + +Short stages keep AdEMAMix's internal warmup fixed to the 256M protocol horizon; +they are true prefixes of the main optimizer dynamics rather than compressed +short-horizon variants. + +## S2: all-optimizer main curves + +```bash +python scripts/sub_one_pass/run_all.py \ + --stage main \ + --lr-scale-file ./exps/sub_one_pass/selected_lr_scales.json \ + $DATA_ARGS \ + --results-dir ./exps/sub_one_pass +``` + +Every run is sequential. `manifest_main.json` is atomically updated after each +state change. Restarting the same command skips directories that already contain +`summary.json`; pass `--rerun-completed` only when an intentional rerun is +required. + +## S3: paired-seed confirmation + +After S2, choose the prespecified leading group plus AdamW and run two additional +seeds. For example: + +```bash +python scripts/sub_one_pass/run_all.py \ + --stage confirm \ + --optimizers adamw muon soap lion \ + --seeds 1 2 \ + --lr-scale-file ./exps/sub_one_pass/selected_lr_scales.json \ + $DATA_ARGS \ + --results-dir ./exps/sub_one_pass +``` + +## Summaries and time estimate + +```bash +python scripts/sub_one_pass/summarize.py ./exps/sub_one_pass + +python scripts/sub_one_pass/estimate_runtime.py \ + ./exps/sub_one_pass \ + --target-tokens 268435456 +``` + +The runtime estimate must be generated from GPU smoke/calibration results. +CPU estimates and generic A800 estimates are not substitutes for measurement. + +## Dry runs + +```bash +python scripts/sub_one_pass/run_all.py --stage smoke --dry-run +python scripts/sub_one_pass/run_all.py --stage tune --dry-run +python scripts/sub_one_pass/run_all.py --stage main --dry-run +``` diff --git a/scripts/sub_one_pass/estimate_runtime.py b/scripts/sub_one_pass/estimate_runtime.py new file mode 100644 index 0000000..9ed9b93 --- /dev/null +++ b/scripts/sub_one_pass/estimate_runtime.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Estimate a serial single-GPU queue from completed calibration summaries.""" + +import argparse +import json +import statistics +from pathlib import Path + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("results_dir", nargs="?", default="./exps/sub_one_pass") + parser.add_argument("--target-tokens", type=int, default=268_435_456) + parser.add_argument("--seeds", type=int, default=1) + parser.add_argument( + "--overhead-factor", + type=float, + default=1.15, + help="Multiplier for evaluation, startup, checkpointing and queue overhead.", + ) + return parser.parse_args() + + +def main(): + args = parse_args() + if args.target_tokens <= 0 or args.seeds <= 0 or args.overhead_factor < 1: + raise ValueError("target-tokens and seeds must be positive; overhead >= 1.") + + throughputs = {} + for path in Path(args.results_dir).rglob("summary.json"): + payload = json.loads(path.read_text()) + optimizer = payload.get("args", {}).get("opt") + throughput = payload.get("mean_tokens_per_second") + if optimizer and throughput and throughput > 0: + throughputs.setdefault(optimizer, []).append(float(throughput)) + + if not throughputs: + raise RuntimeError("No summaries with mean_tokens_per_second were found.") + + optimizer_tps = { + optimizer: statistics.median(values) + for optimizer, values in sorted(throughputs.items()) + } + total_seconds = 0.0 + print("optimizer,tokens_per_second,estimated_hours") + for optimizer, throughput in optimizer_tps.items(): + seconds = args.target_tokens * args.seeds / throughput * args.overhead_factor + total_seconds += seconds + print(f"{optimizer},{throughput:.1f},{seconds / 3600:.3f}") + print(f"TOTAL,,{total_seconds / 3600:.3f}") + print( + "Estimate uses measured training throughput and a global overhead factor; " + "it is not a billing guarantee." + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/sub_one_pass/prepare_fineweb_parquet.py b/scripts/sub_one_pass/prepare_fineweb_parquet.py new file mode 100644 index 0000000..b584190 --- /dev/null +++ b/scripts/sub_one_pass/prepare_fineweb_parquet.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Tokenize local FineWeb parquet shards into fixed uint16 token-bin files.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path + +import numpy as np +import pyarrow.parquet as pq +import tiktoken +from prepare_fineweb_stream import write_tokens + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--source", type=Path, nargs="+", required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--train-tokens", type=int, default=536_870_912) + parser.add_argument("--val-tokens", type=int, default=8_388_608) + parser.add_argument("--batch-rows", type=int, default=1_024) + return parser.parse_args() + + +def documents(paths, batch_rows): + for path in paths: + parquet = pq.ParquetFile(path) + for batch in parquet.iter_batches(columns=["text"], batch_size=batch_rows): + for text in batch.column(0).to_pylist(): + yield {"text": text} + + +def sha256(path, chunk_size=8 * 1024 * 1024): + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(chunk_size), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main(): + args = parse_args() + if args.train_tokens <= 0 or args.val_tokens <= 0: + raise ValueError("Token counts must be positive.") + missing = [str(path) for path in args.source if not path.is_file()] + if missing: + raise FileNotFoundError("Missing parquet sources: " + ", ".join(missing)) + + args.output_dir.mkdir(parents=True, exist_ok=True) + train_final = args.output_dir / "train.bin" + val_final = args.output_dir / "val.bin" + manifest_path = args.output_dir / "manifest.json" + if train_final.exists() or val_final.exists(): + raise FileExistsError(f"{args.output_dir} already contains token files.") + train_partial = train_final.with_suffix(".bin.partial") + val_partial = val_final.with_suffix(".bin.partial") + + tokenizer = tiktoken.get_encoding("gpt2") + train_memmap = np.memmap( + train_partial, dtype=np.uint16, mode="w+", shape=(args.train_tokens,) + ) + val_memmap = np.memmap( + val_partial, dtype=np.uint16, mode="w+", shape=(args.val_tokens,) + ) + document_count, boundary_discarded = write_tokens( + documents(args.source, args.batch_rows), + tokenizer, + val_memmap, + train_memmap, + ) + train_memmap.flush() + val_memmap.flush() + del train_memmap, val_memmap + os.replace(train_partial, train_final) + os.replace(val_partial, val_final) + + sources = [ + { + "path": str(path.resolve()), + "size_bytes": path.stat().st_size, + "sha256": sha256(path), + } + for path in args.source + ] + manifest = { + "dataset": "HuggingFaceFW/fineweb", + "config": "sample-10BT", + "sources": sources, + "tokenizer": "gpt2", + "train_tokens": args.train_tokens, + "val_tokens": args.val_tokens, + "documents_consumed": document_count, + "split_unit": "source_document", + "validation_boundary_discarded_tokens": boundary_discarded, + } + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + print(json.dumps(manifest, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/sub_one_pass/prepare_fineweb_stream.py b/scripts/sub_one_pass/prepare_fineweb_stream.py new file mode 100644 index 0000000..29cfd19 --- /dev/null +++ b/scripts/sub_one_pass/prepare_fineweb_stream.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Stream a fixed-size FineWeb corpus directly into uint16 token files.""" + +import argparse +import json +import os +from pathlib import Path + +import numpy as np + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--output-dir", required=True) + parser.add_argument("--config", default="sample-10BT") + parser.add_argument("--train-tokens", type=int, default=1_073_741_824) + parser.add_argument("--val-tokens", type=int, default=8_388_608) + parser.add_argument("--shuffle-buffer", type=int, default=10_000) + parser.add_argument("--seed", type=int, default=2357) + return parser.parse_args() + + +def write_tokens(stream, tokenizer, val_memmap, train_memmap): + val_written = 0 + train_written = 0 + validation_boundary_discarded_tokens = 0 + for document_count, example in enumerate(stream, start=1): + token_ids = tokenizer.encode_ordinary(example["text"]) + token_ids.append(tokenizer.eot_token) + tokens = np.asarray(token_ids, dtype=np.uint16) + + if val_written < len(val_memmap): + count = min(len(tokens), len(val_memmap) - val_written) + val_memmap[val_written : val_written + count] = tokens[:count] + val_written += count + if val_written == len(val_memmap): + # Never split one source document across validation and train. + validation_boundary_discarded_tokens = len(tokens) - count + tokens = np.empty(0, dtype=np.uint16) + elif train_written < len(train_memmap): + count = min(len(tokens), len(train_memmap) - train_written) + train_memmap[train_written : train_written + count] = tokens[:count] + train_written += count + + if document_count % 1_000 == 0: + print( + f"documents={document_count:,} " + f"val_tokens={val_written:,}/{len(val_memmap):,} " + f"train_tokens={train_written:,}/{len(train_memmap):,}", + flush=True, + ) + val_memmap.flush() + train_memmap.flush() + + if val_written == len(val_memmap) and train_written == len(train_memmap): + return document_count, validation_boundary_discarded_tokens + + raise RuntimeError( + "FineWeb stream ended before the requested token budgets were filled." + ) + + +def main(): + import tiktoken + from datasets import load_dataset + + args = parse_args() + if args.train_tokens <= 0 or args.val_tokens <= 0: + raise ValueError("Token counts must be positive.") + + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + train_final = output_dir / "train.bin" + val_final = output_dir / "val.bin" + manifest_path = output_dir / "manifest.json" + if train_final.exists() or val_final.exists(): + raise FileExistsError( + f"{output_dir} already contains train.bin or val.bin; " + "choose a new output directory." + ) + + train_partial = output_dir / "train.bin.partial" + val_partial = output_dir / "val.bin.partial" + tokenizer = tiktoken.get_encoding("gpt2") + stream = load_dataset( + "HuggingFaceFW/fineweb", + name=args.config, + split="train", + streaming=True, + ) + stream = stream.shuffle(seed=args.seed, buffer_size=args.shuffle_buffer) + + train_memmap = np.memmap( + train_partial, dtype=np.uint16, mode="w+", shape=(args.train_tokens,) + ) + val_memmap = np.memmap( + val_partial, dtype=np.uint16, mode="w+", shape=(args.val_tokens,) + ) + ( + document_count, + validation_boundary_discarded_tokens, + ) = write_tokens(stream, tokenizer, val_memmap, train_memmap) + train_memmap.flush() + val_memmap.flush() + del train_memmap, val_memmap + os.replace(train_partial, train_final) + os.replace(val_partial, val_final) + + manifest = { + "dataset": "HuggingFaceFW/fineweb", + "config": args.config, + "seed": args.seed, + "shuffle_buffer": args.shuffle_buffer, + "tokenizer": "gpt2", + "train_tokens": args.train_tokens, + "val_tokens": args.val_tokens, + "documents_consumed": document_count, + "split_unit": "source_document", + "validation_boundary_discarded_tokens": (validation_boundary_discarded_tokens), + } + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + print(json.dumps(manifest, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/sub_one_pass/prepare_preflight_data.py b/scripts/sub_one_pass/prepare_preflight_data.py new file mode 100644 index 0000000..255f05f --- /dev/null +++ b/scripts/sub_one_pass/prepare_preflight_data.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Create a small deterministic token-bin for optimizer compatibility checks.""" + +import argparse +import json +from pathlib import Path + +import numpy as np + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--output-dir", required=True) + parser.add_argument("--train-tokens", type=int, default=4_194_304) + parser.add_argument("--val-tokens", type=int, default=524_288) + parser.add_argument("--vocab-size", type=int, default=50_304) + parser.add_argument("--seed", type=int, default=20260728) + return parser.parse_args() + + +def write_random_tokens(path, count, vocab_size, rng, chunk_size=1_048_576): + token_file = np.memmap(path, dtype=np.uint16, mode="w+", shape=(count,)) + for start in range(0, count, chunk_size): + end = min(start + chunk_size, count) + token_file[start:end] = rng.integers( + 0, vocab_size, size=end - start, dtype=np.uint16 + ) + token_file.flush() + del token_file + + +def main(): + args = parse_args() + if args.train_tokens <= 0 or args.val_tokens <= 0: + raise ValueError("Token counts must be positive.") + if not 1 < args.vocab_size <= np.iinfo(np.uint16).max: + raise ValueError("vocab-size must fit uint16 and be greater than one.") + + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + train_path = output_dir / "train.bin" + val_path = output_dir / "val.bin" + manifest_path = output_dir / "manifest.json" + if train_path.exists() or val_path.exists(): + raise FileExistsError(f"{output_dir} already contains token files.") + + rng = np.random.default_rng(args.seed) + write_random_tokens(train_path, args.train_tokens, args.vocab_size, rng) + write_random_tokens(val_path, args.val_tokens, args.vocab_size, rng) + manifest = { + "purpose": "optimizer-compatibility-preflight-only", + "synthetic": True, + "seed": args.seed, + "vocab_size": args.vocab_size, + "train_tokens": args.train_tokens, + "val_tokens": args.val_tokens, + "eligible_for_scientific_comparison": False, + } + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + print(json.dumps(manifest, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/sub_one_pass/run_all.py b/scripts/sub_one_pass/run_all.py new file mode 100644 index 0000000..8137e8a --- /dev/null +++ b/scripts/sub_one_pass/run_all.py @@ -0,0 +1,598 @@ +#!/usr/bin/env python3 +"""Run every optimizer through one nested, single-GPU token-budget trajectory.""" + +import argparse +import json +import shutil +import subprocess +import sys +import time +from pathlib import Path + +STAGE_EVAL_TOKENS = { + "smoke": [2_097_152], + "tune": [16_777_216], + "main": [ + 16_777_216, + 33_554_432, + 67_108_864, + 134_217_728, + 268_435_456, + ], + "confirm": [ + 16_777_216, + 33_554_432, + 67_108_864, + 134_217_728, + 268_435_456, + ], +} + +OPTIMIZER_ARGS = { + "adamw": ["--lr", "1e-3", "--beta1", "0.8", "--beta2", "0.999"], + "sgd": ["--lr", "1e-2", "--momentum", "0.9"], + "muon": [ + "--lr", + "1e-3", + "--muon_lr_factor", + "1e-2", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--momentum", + "0.95", + "--nesterov", + "True", + ], + "soap": ["--lr", "1e-3", "--beta1", "0.9", "--beta2", "0.999"], + "ademamix": [ + "--lr", + "1e-3", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--adema_beta3", + "0.999", + "--adema_alpha", + "8.0", + ], + "lion": ["--lr", "1e-3", "--beta1", "0.9", "--beta2", "0.99"], + "sf-adamw": [ + "--lr", + "1e-3", + "--beta1", + "0.9", + "--beta2", + "0.9999", + "--scheduler", + "none", + ], + "sf-sgd": [ + "--lr", + "1e-2", + "--momentum", + "0.9", + "--scheduler", + "none", + ], + "signsgd": ["--lr", "1e-3"], + "signum": ["--lr", "1e-3", "--momentum", "0.9"], + "prodigy": [ + "--lr", + "1.0", + "--beta1", + "0.9", + "--beta2", + "0.999", + "--prodigy_use_bias_correction", + "True", + ], + "sophiag": ["--lr", "1e-3", "--beta1", "0.9", "--beta2", "0.999"], + "adopt": ["--lr", "1e-3", "--beta1", "0.9", "--beta2", "0.999"], + "mars": [ + "--lr", + "1e-3", + "--mars_lr", + "3e-3", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--mars_beta1", + "0.95", + "--mars_beta2", + "0.99", + ], + "adafactor": ["--lr", "1e-3", "--beta1", "0.9"], + "lamb": ["--lr", "1e-3", "--beta1", "0.9", "--beta2", "0.999"], + "scion": ["--lr", "1e-3", "--momentum", "0.9"], + "scion-light": ["--lr", "1e-3", "--momentum", "0.9"], + "d-muon": [ + "--lr", + "1e-3", + "--beta1", + "0.8", + "--beta2", + "0.999", + "--momentum", + "0.95", + "--nesterov", + "True", + ], + "muon-pytorch": [ + "--lr", + "2e-2", + "--muon_adamw_lr", + "1e-3", + "--momentum", + "0.95", + "--nesterov", + "True", + ], +} + +LR_FLAGS = { + "adamw": {"--lr"}, + "sgd": {"--lr"}, + "muon": {"--lr", "--muon_lr_factor"}, + "soap": {"--lr"}, + "ademamix": {"--lr"}, + "lion": {"--lr"}, + "sf-adamw": {"--lr"}, + "sf-sgd": {"--lr"}, + "signsgd": {"--lr"}, + "signum": {"--lr"}, + "prodigy": {"--lr"}, + "sophiag": {"--lr"}, + "adopt": {"--lr"}, + "mars": {"--lr", "--mars_lr"}, + "adafactor": {"--lr"}, + "lamb": {"--lr"}, + "scion": {"--lr"}, + "scion-light": {"--lr"}, + "d-muon": {"--lr"}, + "muon-pytorch": {"--lr", "--muon_adamw_lr"}, +} + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Sequentially benchmark all repository optimizers on one GPU." + ) + parser.add_argument( + "--optimizers", + nargs="+", + default=["all"], + help="Optimizer names or 'all'.", + ) + parser.add_argument("--dataset", default="fineweb") + parser.add_argument("--datasets-dir", default="./src/data/datasets/") + parser.add_argument("--train-data-path", default=None) + parser.add_argument("--val-data-path", default=None) + parser.add_argument("--results-dir", default="./exps/sub_one_pass") + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--seed", type=int, default=0) + parser.add_argument( + "--seeds", + type=int, + nargs="+", + default=None, + help="Run multiple model seeds. Overrides --seed.", + ) + parser.add_argument("--data-seed", type=int, default=1337) + parser.add_argument( + "--stage", + choices=sorted(STAGE_EVAL_TOKENS), + default="main", + help="Use the default token budget and LR grid for this protocol stage.", + ) + parser.add_argument( + "--eval-tokens", + type=int, + nargs="+", + default=None, + help="Override the selected stage's token checkpoints.", + ) + parser.add_argument( + "--protocol-horizon-tokens", + type=int, + default=268_435_456, + help="Fixed horizon for optimizer-internal schedules such as AdEMAMix.", + ) + parser.add_argument( + "--lr-scales", + type=float, + nargs="+", + default=None, + help="Scale all learning-rate fields. Tune defaults to 0.3, 1, 3.", + ) + parser.add_argument( + "--lr-scale-file", + default=None, + help="JSON object mapping optimizer names to selected LR scales.", + ) + parser.add_argument("--batch-size", type=int, default=16) + parser.add_argument("--acc-steps", type=int, default=8) + parser.add_argument("--sequence-length", type=int, default=512) + parser.add_argument("--n-layer", type=int, default=8) + parser.add_argument("--n-head", type=int, default=6) + parser.add_argument("--n-embd", type=int, default=384) + parser.add_argument("--warmup-steps", type=int, default=16) + parser.add_argument("--eval-batches", type=int, default=64) + parser.add_argument("--weight-decay", type=float, default=0.1) + parser.add_argument("--grad-clip", type=float, default=0.5) + parser.add_argument("--wandb", action="store_true") + parser.add_argument("--wandb-project", default="llm-optimizer-sub-one-pass") + parser.add_argument("--wandb-entity", default=None) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--fail-fast", action="store_true") + parser.add_argument( + "--rerun-completed", + action="store_true", + help="Run even when the experiment directory already has summary.json.", + ) + return parser.parse_args() + + +def validate_args(args): + if args.eval_tokens is None: + args.eval_tokens = list(STAGE_EVAL_TOKENS[args.stage]) + if args.seeds is None: + args.seeds = [args.seed] + if len(args.seeds) != len(set(args.seeds)): + raise ValueError("--seeds must be unique.") + if args.lr_scales is None: + args.lr_scales = [0.3, 1.0, 3.0] if args.stage == "tune" else [1.0] + if any(value <= 0 for value in args.lr_scales): + raise ValueError("--lr-scales values must be positive.") + if len(args.lr_scales) != len(set(args.lr_scales)): + raise ValueError("--lr-scales must be unique.") + + unknown = set(args.optimizers) - set(OPTIMIZER_ARGS) - {"all"} + if unknown: + raise ValueError(f"Unknown optimizers: {sorted(unknown)}") + if "all" in args.optimizers and len(args.optimizers) != 1: + raise ValueError("'all' cannot be combined with explicit optimizer names.") + if not args.eval_tokens: + raise ValueError("--eval-tokens must contain at least one token boundary.") + if any(value <= 0 for value in args.eval_tokens): + raise ValueError("--eval-tokens values must be positive.") + if args.eval_tokens != sorted(set(args.eval_tokens)): + raise ValueError("--eval-tokens must be strictly increasing and unique.") + + tokens_per_step = args.batch_size * args.acc_steps * args.sequence_length + invalid = [value for value in args.eval_tokens if value % tokens_per_step] + if invalid: + raise ValueError( + f"Token boundaries {invalid} are not divisible by " + f"tokens_per_step={tokens_per_step}." + ) + if args.protocol_horizon_tokens < args.eval_tokens[-1]: + raise ValueError("--protocol-horizon-tokens must cover the largest checkpoint.") + if args.protocol_horizon_tokens % tokens_per_step: + raise ValueError( + "--protocol-horizon-tokens must be divisible by " + f"tokens_per_step={tokens_per_step}." + ) + if args.n_embd % args.n_head: + raise ValueError("--n-embd must be divisible by --n-head.") + if args.dataset == "token-bin" and not ( + args.train_data_path and args.val_data_path + ): + raise ValueError( + "--dataset token-bin requires --train-data-path and --val-data-path." + ) + if args.lr_scale_file is not None and len(args.lr_scales) != 1: + raise ValueError( + "--lr-scale-file cannot be combined with multiple --lr-scales." + ) + + +def load_lr_scale_map(path): + if path is None: + return {} + payload = json.loads(Path(path).read_text()) + if not isinstance(payload, dict): + raise ValueError("--lr-scale-file must contain a JSON object.") + unknown = set(payload) - set(OPTIMIZER_ARGS) + if unknown: + raise ValueError(f"Unknown optimizers in LR scale file: {sorted(unknown)}") + scales = {} + for optimizer, value in payload.items(): + value = float(value) + if value <= 0: + raise ValueError(f"LR scale for {optimizer} must be positive.") + scales[optimizer] = value + return scales + + +def scaled_optimizer_args(optimizer, scale): + arguments = list(OPTIMIZER_ARGS[optimizer]) + lr_flags = LR_FLAGS[optimizer] + for index in range(0, len(arguments), 2): + if arguments[index] in lr_flags: + arguments[index + 1] = f"{float(arguments[index + 1]) * scale:.12g}" + return arguments + + +def format_scale(scale): + return f"{scale:.6g}".replace("-", "m").replace(".", "p") + + +def command_for(args, optimizer, repo_root, seed=None, lr_scale=1.0): + seed = args.seed if seed is None else seed + max_tokens = args.eval_tokens[-1] + steps = max_tokens // (args.batch_size * args.acc_steps * args.sequence_length) + protocol_horizon_steps = args.protocol_horizon_tokens // ( + args.batch_size * args.acc_steps * args.sequence_length + ) + experiment_name = ( + f"sub1_{args.stage}_{args.dataset}_{optimizer}_seed{seed}_" + f"lrscale{format_scale(lr_scale)}_tokens{max_tokens}" + ) + + if optimizer == "d-muon": + environment_torchrun = Path(sys.executable).with_name("torchrun") + torchrun = ( + str(environment_torchrun) + if environment_torchrun.is_file() + else shutil.which("torchrun") + ) + if torchrun is None: + raise RuntimeError("d-muon requires torchrun, but it is not on PATH.") + command = [ + torchrun, + "--standalone", + "--nproc_per_node=1", + str(repo_root / "src" / "main.py"), + "--distributed_backend", + "nccl", + ] + else: + command = [sys.executable, str(repo_root / "src" / "main.py")] + + command += [ + "--config_format", + "base", + "--model", + "llama", + "--dataset", + args.dataset, + "--datasets_dir", + args.datasets_dir, + "--device", + args.device, + "--n_layer", + str(args.n_layer), + "--n_head", + str(args.n_head), + "--n_embd", + str(args.n_embd), + "--batch_size", + str(args.batch_size), + "--acc_steps", + str(args.acc_steps), + "--sequence_length", + str(args.sequence_length), + "--opt", + optimizer, + "--train_token_budget", + str(max_tokens), + "--eval_at_tokens", + *[str(value) for value in args.eval_tokens], + "--strict_sub_one_pass", + "--fixed_data_boundaries", + "--lazy_data_permutation", + "--scheduler", + "warmup_constant", + "--warmup_steps", + str(args.warmup_steps), + "--eval_interval", + "0", + "--eval_batches", + str(args.eval_batches), + "--limit_final_eval", + "--latest_ckpt_interval", + str(steps), + "--log_interval", + "50", + "--weight_decay", + str(args.weight_decay), + "--grad_clip", + str(args.grad_clip), + "--seed", + str(seed), + "--data_seed", + str(args.data_seed), + "--results_base_folder", + args.results_dir, + "--experiment_name", + experiment_name, + ] + command += scaled_optimizer_args(optimizer, lr_scale) + if args.dataset == "token-bin": + command += [ + "--train_data_path", + args.train_data_path, + "--val_data_path", + args.val_data_path, + ] + + # Keep short stages prefix-compatible with the main trajectory. + if optimizer == "ademamix": + command += [ + "--adema_beta3_warmup", + str(protocol_horizon_steps), + "--adema_alpha_warmup", + str(protocol_horizon_steps), + ] + + if args.wandb: + command += ["--wandb", "--wandb_project", args.wandb_project] + if args.wandb_entity: + command += ["--wandb_entity", args.wandb_entity] + return command + + +def write_manifest(path, manifest): + temporary_path = path.with_suffix(path.suffix + ".tmp") + temporary_path.write_text(json.dumps(manifest, indent=2) + "\n") + temporary_path.replace(path) + + +def initialize_manifest(path, args, optimizers): + invocation = {"arguments": vars(args), "started_at_unix": time.time()} + if path.is_file(): + manifest = json.loads(path.read_text()) + if ( + manifest.get("protocol") != "sub-one-pass" + or manifest.get("stage") != args.stage + ): + raise ValueError(f"{path} belongs to a different experiment protocol.") + manifest["optimizers"] = list( + dict.fromkeys(manifest.get("optimizers", []) + optimizers) + ) + manifest.setdefault("invocations", []).append(invocation) + manifest.setdefault("runs", []) + return manifest + return { + "protocol": "sub-one-pass", + "stage": args.stage, + "optimizers": optimizers, + "invocations": [invocation], + "runs": [], + } + + +def upsert_run_record(manifest, run_record): + for existing in manifest["runs"]: + if existing.get("experiment_dir") == run_record["experiment_dir"]: + existing.clear() + existing.update(run_record) + return existing + manifest["runs"].append(run_record) + return run_record + + +def run_and_tee(command, repo_root, log_path): + log_path.parent.mkdir(parents=True, exist_ok=True) + with log_path.open("a") as log_handle: + process = subprocess.Popen( + command, + cwd=repo_root, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + for line in process.stdout: + print(line, end="", flush=True) + log_handle.write(line) + log_handle.flush() + return process.wait() + + +def normalize_paths(args): + args.results_dir = str(Path(args.results_dir).resolve()) + args.datasets_dir = str(Path(args.datasets_dir).resolve()) + if args.train_data_path is not None: + args.train_data_path = str(Path(args.train_data_path).resolve()) + if args.val_data_path is not None: + args.val_data_path = str(Path(args.val_data_path).resolve()) + if args.lr_scale_file is not None: + args.lr_scale_file = str(Path(args.lr_scale_file).resolve()) + + +def main(): + args = parse_args() + validate_args(args) + normalize_paths(args) + repo_root = Path(__file__).resolve().parents[2] + optimizers = list(OPTIMIZER_ARGS) if args.optimizers == ["all"] else args.optimizers + lr_scale_map = load_lr_scale_map(args.lr_scale_file) + missing_scales = set(optimizers) - set(lr_scale_map) + if args.lr_scale_file is not None and missing_scales: + raise ValueError( + "LR scale file is missing selected optimizers: " + + ", ".join(sorted(missing_scales)) + ) + + results_dir = Path(args.results_dir) + if not args.dry_run: + results_dir.mkdir(parents=True, exist_ok=True) + manifest_path = results_dir / f"manifest_{args.stage}.json" + manifest = initialize_manifest(manifest_path, args, optimizers) + write_manifest(manifest_path, manifest) + else: + manifest = { + "protocol": "sub-one-pass", + "stage": args.stage, + "optimizers": optimizers, + "invocations": [{"arguments": vars(args)}], + "runs": [], + } + + stop = False + for seed in args.seeds: + for optimizer in optimizers: + scales = [lr_scale_map.get(optimizer, value) for value in args.lr_scales] + for lr_scale in scales: + command = command_for(args, optimizer, repo_root, seed, lr_scale) + experiment_name = command[command.index("--experiment_name") + 1] + experiment_dir = results_dir / experiment_name + summary_path = experiment_dir / "summary.json" + log_path = experiment_dir / "run.log" + print( + f"\n[{optimizer} seed={seed} lr_scale={lr_scale:g}] " + f"{' '.join(command)}", + flush=True, + ) + run_record = { + "optimizer": optimizer, + "seed": seed, + "lr_scale": lr_scale, + "command": command, + "experiment_dir": str(experiment_dir), + } + + if args.dry_run: + run_record["status"] = "dry-run" + manifest["runs"].append(run_record) + continue + + run_record = upsert_run_record(manifest, run_record) + if summary_path.is_file() and not args.rerun_completed: + run_record["status"] = "skipped-completed" + write_manifest(manifest_path, manifest) + continue + + run_record["status"] = "running" + run_record["started_at_unix"] = time.time() + write_manifest(manifest_path, manifest) + returncode = run_and_tee(command, repo_root, log_path) + run_record["returncode"] = returncode + run_record["finished_at_unix"] = time.time() + run_record["status"] = "completed" if returncode == 0 else "failed" + write_manifest(manifest_path, manifest) + if returncode != 0 and args.fail_fast: + stop = True + break + if stop: + break + if stop: + break + + failed = [run for run in manifest["runs"] if run["status"] == "failed"] + if failed: + print( + "Failed optimizers: " + ", ".join(run["optimizer"] for run in failed), + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/sub_one_pass/run_confirm_after_main.py b/scripts/sub_one_pass/run_confirm_after_main.py new file mode 100644 index 0000000..3b46e47 --- /dev/null +++ b/scripts/sub_one_pass/run_confirm_after_main.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""Wait for S2, select three challengers, and run paired-seed S3 confirmation.""" + +import argparse +import json +import math +import subprocess +import sys +import time +from pathlib import Path + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--repo-root", type=Path, required=True) + parser.add_argument("--train-data-path", type=Path, required=True) + parser.add_argument("--val-data-path", type=Path, required=True) + parser.add_argument("--results-dir", type=Path, required=True) + parser.add_argument("--lr-scale-file", type=Path, required=True) + parser.add_argument("--poll-seconds", type=float, default=15.0) + return parser.parse_args() + + +def write_json_atomic(path, payload): + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(payload, indent=2) + "\n") + temporary.replace(path) + + +def write_state(path, stage, **extra): + write_json_atomic( + path, + {"stage": stage, "updated_at_unix": time.time(), **extra}, + ) + + +def wait_for_main(state_path, poll_seconds): + while True: + if state_path.is_file(): + state = json.loads(state_path.read_text()) + stage = state.get("stage") + if stage == "completed": + return + if stage == "failed": + raise RuntimeError("S2 failed; S3 will not be started.") + time.sleep(poll_seconds) + + +def select_challengers(results_dir): + manifest_path = results_dir / "manifest_main.json" + manifest = json.loads(manifest_path.read_text()) + scores = [] + for run in manifest.get("runs", []): + if run.get("status") not in {"completed", "skipped-completed"}: + continue + summary_path = Path(run["experiment_dir"]) / "summary.json" + summary = json.loads(summary_path.read_text()) + losses = summary.get("val_loss", []) + if not losses: + raise ValueError(f"No validation loss in {summary_path}") + final_loss = float(losses[-1]) + if not math.isfinite(final_loss): + raise ValueError(f"Non-finite final validation loss in {summary_path}") + scores.append( + { + "optimizer": run["optimizer"], + "final_val_loss": final_loss, + "summary_path": str(summary_path), + } + ) + + by_optimizer = {} + for score in scores: + previous = by_optimizer.get(score["optimizer"]) + if previous is None or score["final_val_loss"] < previous["final_val_loss"]: + by_optimizer[score["optimizer"]] = score + if len(by_optimizer) != 20: + raise ValueError( + f"Expected 20 completed S2 optimizers, found {len(by_optimizer)}." + ) + + ranking = sorted( + by_optimizer.values(), + key=lambda item: (item["final_val_loss"], item["optimizer"]), + ) + challengers = [item["optimizer"] for item in ranking if item["optimizer"] != "adamw"][ + :3 + ] + return ["adamw", *challengers], ranking + + +def run(command, cwd): + print("$ " + " ".join(str(value) for value in command), flush=True) + return subprocess.run(command, cwd=cwd, check=False).returncode + + +def manifest_counts(path): + if not path.is_file(): + return {} + manifest = json.loads(path.read_text()) + counts = {} + for record in manifest.get("runs", []): + status = record.get("status", "unknown") + counts[status] = counts.get(status, 0) + 1 + return counts + + +def main(): + args = parse_args() + args.repo_root = args.repo_root.resolve() + args.results_dir = args.results_dir.resolve() + args.results_dir.mkdir(parents=True, exist_ok=True) + state_path = args.results_dir / "pipeline_state.json" + + try: + wait_for_main(state_path, args.poll_seconds) + write_state(state_path, "confirm_select") + optimizers, ranking = select_challengers(args.results_dir) + selection_path = args.results_dir / "confirm_selection.json" + write_json_atomic( + selection_path, + { + "selection_rule": ( + "AdamW baseline plus the three non-AdamW optimizers with the " + "lowest S2 final validation loss at 268435456 tokens." + ), + "selected_optimizers": optimizers, + "seeds": [1, 2], + "ranking": ranking, + "created_at_unix": time.time(), + }, + ) + + write_state( + state_path, + "confirm", + selected_optimizers=optimizers, + seeds=[1, 2], + selection_file=str(selection_path), + ) + returncode = run( + [ + sys.executable, + str(args.repo_root / "scripts/sub_one_pass/run_all.py"), + "--stage", + "confirm", + "--optimizers", + *optimizers, + "--seeds", + "1", + "2", + "--lr-scale-file", + str(args.lr_scale_file), + "--dataset", + "token-bin", + "--train-data-path", + str(args.train_data_path), + "--val-data-path", + str(args.val_data_path), + "--results-dir", + str(args.results_dir), + ], + args.repo_root, + ) + if returncode: + write_state( + state_path, + "failed", + failed_stage="confirm", + returncode=returncode, + selected_optimizers=optimizers, + ) + return returncode + + write_state( + state_path, + "confirm_summarize", + selected_optimizers=optimizers, + ) + returncode = run( + [ + sys.executable, + str(args.repo_root / "scripts/sub_one_pass/summarize.py"), + str(args.results_dir), + ], + args.repo_root, + ) + if returncode: + write_state( + state_path, + "failed", + failed_stage="confirm_summarize", + returncode=returncode, + selected_optimizers=optimizers, + ) + return returncode + + write_state( + state_path, + "completed", + s3_completed=True, + selected_optimizers=optimizers, + seeds=[1, 2], + main_manifest_counts=manifest_counts( + args.results_dir / "manifest_main.json" + ), + confirm_manifest_counts=manifest_counts( + args.results_dir / "manifest_confirm.json" + ), + selection_file=str(selection_path), + summary_csv=str(args.results_dir / "sub_one_pass_results.csv"), + ) + return 0 + except Exception as error: + current_stage = None + if state_path.is_file(): + current_stage = json.loads(state_path.read_text()).get("stage") + if current_stage != "failed": + write_state( + state_path, + "failed", + failed_stage="confirm_gate", + error=f"{type(error).__name__}: {error}", + ) + print(f"{type(error).__name__}: {error}", file=sys.stderr, flush=True) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/sub_one_pass/run_main_stage.py b/scripts/sub_one_pass/run_main_stage.py new file mode 100644 index 0000000..8fc6239 --- /dev/null +++ b/scripts/sub_one_pass/run_main_stage.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Resume a server experiment at the main stage after manual LR acceptance.""" + +import argparse +import json +import subprocess +import sys +import time +from pathlib import Path + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--repo-root", type=Path, required=True) + parser.add_argument("--train-data-path", type=Path, required=True) + parser.add_argument("--val-data-path", type=Path, required=True) + parser.add_argument("--results-dir", type=Path, required=True) + parser.add_argument("--lr-scale-file", type=Path, required=True) + parser.add_argument( + "--accepted-boundary-optimizer", + action="append", + default=[], + ) + return parser.parse_args() + + +def write_state(path, stage, **extra): + payload = {"stage": stage, "updated_at_unix": time.time(), **extra} + temporary = path.with_suffix(".tmp") + temporary.write_text(json.dumps(payload, indent=2) + "\n") + temporary.replace(path) + + +def run(command, cwd): + print("$ " + " ".join(str(value) for value in command), flush=True) + return subprocess.run(command, cwd=cwd, check=False).returncode + + +def main(): + args = parse_args() + args.repo_root = args.repo_root.resolve() + args.results_dir.mkdir(parents=True, exist_ok=True) + state_path = args.results_dir / "pipeline_state.json" + write_state( + state_path, + "main", + lr_scale_file=str(args.lr_scale_file), + accepted_boundary_optimizers=args.accepted_boundary_optimizer, + ) + returncode = run( + [ + sys.executable, + str(args.repo_root / "scripts/sub_one_pass/run_all.py"), + "--stage", + "main", + "--lr-scale-file", + str(args.lr_scale_file), + "--dataset", + "token-bin", + "--train-data-path", + str(args.train_data_path), + "--val-data-path", + str(args.val_data_path), + "--results-dir", + str(args.results_dir), + ], + args.repo_root, + ) + if returncode: + write_state(state_path, "failed", returncode=returncode) + return returncode + + write_state(state_path, "summarize") + returncode = run( + [ + sys.executable, + str(args.repo_root / "scripts/sub_one_pass/summarize.py"), + str(args.results_dir), + ], + args.repo_root, + ) + if returncode: + write_state(state_path, "failed", returncode=returncode) + return returncode + write_state(state_path, "completed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/sub_one_pass/run_server_pipeline.py b/scripts/sub_one_pass/run_server_pipeline.py new file mode 100644 index 0000000..87f718e --- /dev/null +++ b/scripts/sub_one_pass/run_server_pipeline.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""Run tune and main stages on a server after tokenization completes.""" + +import argparse +import json +import subprocess +import sys +import time +from pathlib import Path + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--repo-root", type=Path, required=True) + parser.add_argument("--train-data-path", type=Path, required=True) + parser.add_argument("--val-data-path", type=Path, required=True) + parser.add_argument("--data-manifest", type=Path, required=True) + parser.add_argument("--tokenizer-pid", type=int, default=None) + parser.add_argument("--results-dir", type=Path, required=True) + parser.add_argument("--poll-seconds", type=int, default=15) + parser.add_argument("--data-timeout-hours", type=float, default=6.0) + parser.add_argument("--max-lr-expansions", type=int, default=2) + return parser.parse_args() + + +def process_alive(pid): + try: + Path(f"/proc/{pid}").stat() + return True + except FileNotFoundError: + return False + + +def write_state(path, stage, **extra): + payload = {"stage": stage, "updated_at_unix": time.time(), **extra} + temporary = path.with_suffix(".tmp") + temporary.write_text(json.dumps(payload, indent=2) + "\n") + temporary.replace(path) + + +def wait_for_data(args, state_path): + deadline = time.monotonic() + args.data_timeout_hours * 3600 + while time.monotonic() < deadline: + if ( + args.data_manifest.is_file() + and args.train_data_path.is_file() + and args.val_data_path.is_file() + ): + return + if args.tokenizer_pid is not None and not process_alive(args.tokenizer_pid): + raise RuntimeError( + "Tokenizer exited before producing the data manifest. " + "Inspect the tokenization log; the pipeline will not retry it." + ) + write_state(state_path, "waiting-for-data") + time.sleep(args.poll_seconds) + raise TimeoutError("Timed out waiting for tokenized data.") + + +def run(command, cwd): + print("$ " + " ".join(str(value) for value in command), flush=True) + completed = subprocess.run(command, cwd=cwd, check=False) + if completed.returncode: + raise RuntimeError( + f"Command exited with status {completed.returncode}; not retrying." + ) + + +def common_runner_args(args): + return [ + "--dataset", + "token-bin", + "--train-data-path", + str(args.train_data_path), + "--val-data-path", + str(args.val_data_path), + "--results-dir", + str(args.results_dir), + ] + + +def select_and_expand_lr(args, state_path): + selection_script = args.repo_root / "scripts/sub_one_pass/select_lr.py" + runner_script = args.repo_root / "scripts/sub_one_pass/run_all.py" + tune_manifest = args.results_dir / "manifest_tune.json" + selected_path = args.results_dir / "selected_lr_scales.json" + report_path = args.results_dir / "selected_lr_scales_report.json" + + for expansion_round in range(args.max_lr_expansions + 1): + completed = subprocess.run( + [ + sys.executable, + str(selection_script), + str(tune_manifest), + "--output", + str(selected_path), + ], + cwd=args.repo_root, + check=False, + ) + if completed.returncode == 0: + return selected_path + if completed.returncode != 2: + raise RuntimeError( + f"LR selection exited with status {completed.returncode}." + ) + if expansion_round == args.max_lr_expansions: + raise RuntimeError( + "LR optimum remains on a grid boundary after the allowed " + "expansions. Stopping before main." + ) + + report = json.loads(report_path.read_text()) + boundary_optimizers = [ + optimizer + for optimizer, item in report.items() + if item["selection_on_grid_boundary"] + ] + write_state( + state_path, + "expanding-lr-grid", + expansion_round=expansion_round + 1, + optimizers=boundary_optimizers, + ) + for optimizer in boundary_optimizers: + candidates = report[optimizer]["candidates"] + selected = report[optimizer]["selected_lr_scale"] + scales = sorted(item["lr_scale"] for item in candidates) + new_scale = selected / 3 if selected == scales[0] else selected * 3 + run( + [ + sys.executable, + str(runner_script), + "--stage", + "tune", + "--optimizers", + optimizer, + "--lr-scales", + str(new_scale), + *common_runner_args(args), + ], + args.repo_root, + ) + raise AssertionError("Unreachable") + + +def main(): + args = parse_args() + args.repo_root = args.repo_root.resolve() + args.results_dir.mkdir(parents=True, exist_ok=True) + state_path = args.results_dir / "pipeline_state.json" + runner_script = args.repo_root / "scripts/sub_one_pass/run_all.py" + + write_state(state_path, "waiting-for-data") + wait_for_data(args, state_path) + data_manifest = json.loads(args.data_manifest.read_text()) + if data_manifest.get("train_tokens", 0) <= 268_435_456: + raise ValueError("Formal train corpus must contain more than 256M tokens.") + + write_state(state_path, "tune") + run( + [ + sys.executable, + str(runner_script), + "--stage", + "tune", + *common_runner_args(args), + ], + args.repo_root, + ) + + selected_path = select_and_expand_lr(args, state_path) + write_state(state_path, "main", lr_scale_file=str(selected_path)) + run( + [ + sys.executable, + str(runner_script), + "--stage", + "main", + "--lr-scale-file", + str(selected_path), + *common_runner_args(args), + ], + args.repo_root, + ) + + write_state(state_path, "summarize") + run( + [ + sys.executable, + str(args.repo_root / "scripts/sub_one_pass/summarize.py"), + str(args.results_dir), + ], + args.repo_root, + ) + write_state(state_path, "completed") + + +if __name__ == "__main__": + main() diff --git a/scripts/sub_one_pass/select_lr.py b/scripts/sub_one_pass/select_lr.py new file mode 100644 index 0000000..a9af674 --- /dev/null +++ b/scripts/sub_one_pass/select_lr.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Select one learning-rate scale per optimizer from a tune-stage manifest.""" + +import argparse +import json +from pathlib import Path + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "manifest", + nargs="?", + default="./exps/sub_one_pass/manifest_tune.json", + ) + parser.add_argument( + "--output", + default=None, + help="Defaults to selected_lr_scales.json next to the manifest.", + ) + return parser.parse_args() + + +def final_val_loss(summary_path): + payload = json.loads(summary_path.read_text()) + history = payload.get("eval_history", []) + if not history: + raise ValueError(f"{summary_path} has no eval_history.") + return float(max(history, key=lambda item: item["tokens"])["val_loss"]) + + +def main(): + args = parse_args() + manifest_path = Path(args.manifest) + manifest = json.loads(manifest_path.read_text()) + candidates = {} + missing = [] + + for run in manifest.get("runs", []): + summary_path = Path(run["experiment_dir"]) / "summary.json" + if not summary_path.is_file(): + missing.append(str(summary_path)) + continue + optimizer = run["optimizer"] + candidate = { + "lr_scale": float(run["lr_scale"]), + "val_loss": final_val_loss(summary_path), + "summary": str(summary_path), + } + candidates.setdefault(optimizer, []).append(candidate) + + if missing: + raise RuntimeError( + "Tune stage is incomplete; missing summary files:\n" + "\n".join(missing) + ) + + selected = {} + report = {} + for optimizer in manifest.get("optimizers", []): + optimizer_candidates = sorted( + candidates.get(optimizer, []), key=lambda item: item["lr_scale"] + ) + if not optimizer_candidates: + raise RuntimeError(f"No completed LR candidates for {optimizer}.") + best = min(optimizer_candidates, key=lambda item: item["val_loss"]) + selected[optimizer] = best["lr_scale"] + report[optimizer] = { + "selected_lr_scale": best["lr_scale"], + "selected_val_loss": best["val_loss"], + "selection_on_grid_boundary": best + in (optimizer_candidates[0], optimizer_candidates[-1]), + "candidates": optimizer_candidates, + } + + output_path = ( + Path(args.output) + if args.output is not None + else manifest_path.parent / "selected_lr_scales.json" + ) + output_path.write_text(json.dumps(selected, indent=2, sort_keys=True) + "\n") + report_path = output_path.with_name(output_path.stem + "_report.json") + report_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + boundary = [ + optimizer + for optimizer, item in report.items() + if item["selection_on_grid_boundary"] + ] + print(f"Wrote selected scales to {output_path}") + print(f"Wrote selection report to {report_path}") + if boundary: + print( + "Grid-boundary optima require an expanded LR candidate before main: " + + ", ".join(boundary) + ) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/sub_one_pass/summarize.py b/scripts/sub_one_pass/summarize.py new file mode 100644 index 0000000..a74a6a4 --- /dev/null +++ b/scripts/sub_one_pass/summarize.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Collect sub-one-pass summary.json files into a tidy CSV.""" + +import argparse +import csv +import json +from pathlib import Path + +FIELDS = [ + "optimizer", + "seed", + "data_seed", + "tokens", + "data_exposure", + "iteration", + "val_loss", + "val_perplexity", + "val_accuracy", + "num_eval_batches", + "train_time_seconds", + "mean_tokens_per_second", + "peak_memory_bytes", + "experiment_dir", +] + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("results_dir", nargs="?", default="./exps/sub_one_pass") + parser.add_argument("--output", default=None) + return parser.parse_args() + + +def rows_from_summary(path): + payload = json.loads(path.read_text()) + args = payload.get("args", {}) + for item in payload.get("eval_history", []): + yield { + "optimizer": args.get("opt"), + "seed": args.get("seed"), + "data_seed": args.get("data_seed"), + "tokens": item.get("tokens"), + "data_exposure": item.get("data_exposure"), + "iteration": item.get("iter"), + "val_loss": item.get("val_loss"), + "val_perplexity": item.get("val_perplexity"), + "val_accuracy": item.get("val_accuracy"), + "num_eval_batches": item.get("num_eval_batches"), + "train_time_seconds": payload.get("train_time_seconds"), + "mean_tokens_per_second": payload.get("mean_tokens_per_second"), + "peak_memory_bytes": payload.get("peak_memory_bytes"), + "experiment_dir": str(path.parent), + } + + +def main(): + args = parse_args() + results_dir = Path(args.results_dir) + output = ( + Path(args.output) + if args.output is not None + else results_dir / "sub_one_pass_results.csv" + ) + rows = [] + for path in sorted(results_dir.rglob("summary.json")): + rows.extend(rows_from_summary(path)) + + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=FIELDS) + writer.writeheader() + writer.writerows(rows) + print(f"Wrote {len(rows)} rows to {output}") + + +if __name__ == "__main__": + main() diff --git a/src/config/base.py b/src/config/base.py index 3a2eb8a..5c8eea7 100644 --- a/src/config/base.py +++ b/src/config/base.py @@ -58,7 +58,7 @@ def parse_args(base_parser, args, namespace): parser.add_argument( "--scheduler", default="cos", - choices=["linear", "cos", "wsd", "none", "cos_inf"], + choices=["linear", "cos", "wsd", "none", "cos_inf", "warmup_constant"], ) parser.add_argument( "--final_div_factor", default=1, type=float @@ -68,6 +68,39 @@ def parse_args(base_parser, args, namespace): parser.add_argument("--iterations", default=15000, type=int) parser.add_argument("--warmup_steps", default=3000, type=int) parser.add_argument("--lr", default=1e-3, type=float) + parser.add_argument( + "--train_token_budget", + default=None, + type=int, + help="Override iterations using an exact number of observed training tokens.", + ) + parser.add_argument( + "--eval_at_tokens", + nargs="+", + default=None, + type=int, + help="Evaluate at these cumulative training-token boundaries.", + ) + parser.add_argument( + "--strict_sub_one_pass", + action="store_true", + help="Fail if train_token_budget exceeds the unique tokens in one data pass.", + ) + parser.add_argument( + "--fixed_data_boundaries", + action="store_true", + help="Use fixed non-overlapping sequence boundaries in every data pass.", + ) + parser.add_argument( + "--lazy_data_permutation", + action="store_true", + help="Use a constant-memory deterministic permutation for very large corpora.", + ) + parser.add_argument( + "--limit_final_eval", + action="store_true", + help="Use eval_batches rather than the complete validation set at the final step.", + ) # wsd parser.add_argument("--wsd_final_lr_scale", default=0.0, type=float) parser.add_argument("--wsd_fract_decay", default=0.1, type=float) @@ -125,6 +158,12 @@ def parse_args(base_parser, args, namespace): parser.add_argument("--nesterov", default=False, type=bool) parser.add_argument("--muon_ns_steps", default=5, type=int) parser.add_argument("--muon_lr_factor", default=1.0, type=float) + parser.add_argument( + "--muon_adamw_lr", + default=1e-3, + type=float, + help="AdamW fallback LR for non-Muon parameters in muon-pytorch.", + ) parser.add_argument("--adema_beta3", default=0.9, type=float) parser.add_argument("--adema_alpha", default=2.0, type=float) parser.add_argument("--adema_beta3_warmup", default=None, type=int) @@ -214,6 +253,7 @@ def parse_args(base_parser, args, namespace): default="slimpajama", choices=[ "wikitext", + "token-bin", "shakespeare-char", "arxiv", "arxiv2000", @@ -246,6 +286,8 @@ def parse_args(base_parser, args, namespace): parser.add_argument( "--data_in_ram", action="store_true" ) # force the data to RAM, mostly useless except for openwebtext2 + parser.add_argument("--train_data_path", default=None, type=str) + parser.add_argument("--val_data_path", default=None, type=str) # Model params parser.add_argument( diff --git a/src/data/reader.py b/src/data/reader.py new file mode 100644 index 0000000..ab3ae8d --- /dev/null +++ b/src/data/reader.py @@ -0,0 +1,174 @@ +import math +from pathlib import Path + +import numpy as np +import torch +import torch.distributed as dist + + +class DataReader: + def __init__( + self, + data_src, + batch_size, + sequence_length, + seed=1337, + with_replacement=False, + auto_shard=True, + keep_in_ram=False, + fixed_data_boundaries=False, + lazy_data_permutation=False, + ): + if isinstance(data_src, (str, Path)): + self.data_path = Path(data_src) + self.keep_in_ram = keep_in_ram + if keep_in_ram: + self.data = np.array( + np.memmap(self.data_path, dtype=np.uint16, mode="r") + ) + else: + self.data = None + elif isinstance(data_src, (np.ndarray, np.memmap)): + self.data_path = None + self.data = data_src + self.keep_in_ram = True + else: + raise TypeError( + "data_src must be a path, numpy array, or numpy memmap; " + f"got {type(data_src).__name__}." + ) + + self.batch_size = batch_size + self.sequence_length = sequence_length + self.seed = seed + self.with_replacement = with_replacement + self.fixed_data_boundaries = fixed_data_boundaries + self.lazy_data_permutation = lazy_data_permutation + + self.num_tokens = len(self._get_data()) + + if auto_shard and dist.is_initialized(): + self.world_size = dist.get_world_size() + self.rank = dist.get_rank() + print( + f"Distributed DataReader Initialized for Worker " + f"{self.rank}/{self.world_size}" + ) + else: + self.world_size = 1 + self.rank = 0 + + self.last_epoch = None + self.order = None + self.permutation_offset = None + self.permutation_stride = None + self.epoch_offset = None + self.step = 0 + self.num_batches_of_seqlen = 0 + self.num_sequences = 0 + if not with_replacement: + self._shuffle_epoch(0) + + def __len__(self): + # Extra -1 to have a valid next token for the final start index. + return self.num_tokens - self.sequence_length - 1 + + def _get_data(self): + if self.data is not None: + return self.data + # Construct the memmap each time to avoid a memory leak per NanoGPT. + return np.memmap(self.data_path, dtype=np.uint16, mode="r") + + def __getitem__(self, idx): + assert 0 <= idx < len(self) + data = self._get_data() + x = torch.from_numpy(data[idx : idx + self.sequence_length].astype(np.int64)) + y = torch.from_numpy( + data[idx + 1 : idx + self.sequence_length + 1].astype(np.int64) + ) + return x, y + + def set_step(self, step): + self.step = step + + def sample_batch(self): + data = self._get_data() + if self.with_replacement: + idxs = self._sample_with_replacement(self.step) + else: + idxs = self._sample_without_replacement(self.step) + self.step += 1 + + xy = np.stack([data[i : i + self.sequence_length + 1] for i in idxs]).astype( + np.int64 + ) + x = torch.from_numpy(xy[:, :-1]).contiguous() + y = torch.from_numpy(xy[:, 1:]).contiguous() + return x, y + + def _sample_with_replacement(self, idx): + seed = self.seed + idx * self.world_size + self.rank + rng = np.random.default_rng(seed) + return rng.integers(len(self), self.batch_size) + + def _shuffle_epoch(self, epoch): + seed = self.seed + epoch + rng = np.random.default_rng(seed) + # Drop one sequence so an offset remains valid when boundaries are not fixed. + num_sequences = (len(self)) // self.sequence_length - 1 + if num_sequences < self.batch_size: + raise ValueError( + "Dataset is too small for one complete batch: " + f"{num_sequences} sequences available, batch_size={self.batch_size}." + ) + self.num_sequences = num_sequences + if self.lazy_data_permutation: + # An affine permutation avoids allocating an O(dataset size) index array. + self.order = None + self.permutation_offset = int(rng.integers(num_sequences)) + stride = int(rng.integers(1, num_sequences)) + while math.gcd(stride, num_sequences) != 1: + stride = (stride + 1) % num_sequences + if stride == 0: + stride = 1 + self.permutation_stride = stride + else: + self.order = rng.permutation(num_sequences) + self.epoch_offset = ( + 0 if self.fixed_data_boundaries else rng.integers(self.sequence_length) + ) + self.last_epoch = epoch + self.num_batches_of_seqlen = num_sequences // self.batch_size + + def _sample_without_replacement(self, step): + batch_idx = self.world_size * step + self.rank + epoch_length = self.num_batches_of_seqlen + epoch = batch_idx // epoch_length + if epoch != self.last_epoch: + self._shuffle_epoch(epoch) + epoch_idx = batch_idx % epoch_length + + start = epoch_idx * self.batch_size + end = start + self.batch_size + if self.lazy_data_permutation: + positions = np.arange(start, end, dtype=np.int64) + sequence_ids = ( + self.permutation_offset + positions * self.permutation_stride + ) % self.num_sequences + else: + sequence_ids = self.order[start:end] + return sequence_ids * self.sequence_length + self.epoch_offset + + def num_batches(self): + if self.with_replacement: + return self.num_tokens // self.batch_size + return self.num_batches_of_seqlen + + @property + def unique_tokens_per_epoch(self): + """Target tokens exposed before the no-replacement reader repeats.""" + if self.with_replacement: + raise ValueError( + "unique_tokens_per_epoch is undefined for sampling with replacement." + ) + return self.num_batches_of_seqlen * self.batch_size * self.sequence_length diff --git a/src/data/token_bin.py b/src/data/token_bin.py new file mode 100644 index 0000000..fe28dd3 --- /dev/null +++ b/src/data/token_bin.py @@ -0,0 +1,12 @@ +from pathlib import Path + + +def get_token_bin_data(train_path, val_path): + train_path = Path(train_path) + val_path = Path(val_path) + missing = [str(path) for path in (train_path, val_path) if not path.is_file()] + if missing: + raise FileNotFoundError( + "Missing pre-tokenized uint16 data files: " + ", ".join(missing) + ) + return {"train": str(train_path), "val": str(val_path)} diff --git a/src/data/utils.py b/src/data/utils.py index dde094f..21098cb 100755 --- a/src/data/utils.py +++ b/src/data/utils.py @@ -1,9 +1,6 @@ -from pathlib import Path from typing import Dict import numpy as np -import torch -import torch.distributed as dist from .arxiv import get_arxiv_2000, get_arxiv_full from .benchmarks import SUPPORTED_TASK_MAP @@ -11,9 +8,11 @@ from .fineweb import get_fineweb_data from .fineweb_edu import get_fineweb_edu_data from .openwebtext2 import get_openwebtext2_data +from .reader import DataReader from .redpajama import get_redpajama_data, get_redpajamav2_data from .shakespeare import get_shakespeare_data from .slimpajama import get_slimpajama_data +from .token_bin import get_token_bin_data from .wikitext import get_wikitext_data @@ -22,6 +21,8 @@ def get_dataset(args) -> Dict[str, np.ndarray]: contained in its own python file. The expected format at the moment is a dictionary of np.memmap containing two keys: 'train' and 'val', corresponding to the tokenized training and validation data. """ + if args.dataset == "token-bin": + return get_token_bin_data(args.train_data_path, args.val_data_path) if args.dataset == "wikitext": return get_wikitext_data(args.datasets_dir) if args.dataset == "shakespeare-char": @@ -67,137 +68,3 @@ def get_benchmark_task(name, **kwargs): f"Unknown dataset '{name}'. Supported: {sorted(SUPPORTED_TASK_MAP.keys())}" ) return fn(**kwargs) - - -class DataReader: - def __init__( - self, - data_src, - batch_size, - sequence_length, - seed=1337, - with_replacement=False, - auto_shard=True, - keep_in_ram=False, - ): - if isinstance(data_src, (str, Path)): - self.data_path = Path(data_src) - self.keep_in_ram = keep_in_ram - if keep_in_ram: - self.data = np.array( - np.memmap(self.data_path, dtype=np.uint16, mode="r") - ) - else: - self.data = None - elif isinstance(data_src, (np.ndarray, np.memmap)): - self.data_path = None - self.data = data_src - self.keep_in_ram = True - - self.batch_size = batch_size - self.sequence_length = sequence_length - self.seed = seed - self.with_replacement = with_replacement - - self.num_tokens = len(self._get_data()) - - if auto_shard and dist.is_initialized(): - self.world_size = dist.get_world_size() - self.rank = dist.get_rank() - print( - f"Distributed DataReader Initialized for Worker {self.rank}/{self.world_size}" - ) - else: - self.world_size = 1 - self.rank = 0 - - # Sampling without replacement - self.last_epoch = None - self.order = None - self.epoch_offset = None - self.step = 0 - self.num_batches_of_seqlen = 0 - if not with_replacement: - self._shuffle_epoch(0) - - def __len__(self): - # Length in valid start indices for a sequence - # Extra -1 to have a valid next token for the final token of the last idx - return self.num_tokens - self.sequence_length - 1 - - def _get_data(self): - if self.data is not None: - return self.data - else: - # Construct the memmap each time to avoid a memory leak per NanoGPT - # https://stackoverflow.com/questions/45132940/numpy-memmap-memory-usage-want-to-iterate-once/61472122#61472122 - return np.memmap(self.data_path, dtype=np.uint16, mode="r") - - def __getitem__(self, idx): - # Return the underlying datapoint, no random sampling, no worker sharding - assert 0 <= idx < len(self) - data = self._get_data() - x = torch.from_numpy(data[idx : idx + self.sequence_length].astype(np.int64)) - y = torch.from_numpy( - data[idx + 1 : idx + self.sequence_length + 1].astype(torch.int64) - ) - return x, y - - def set_step(self, step): - self.step = step - - def sample_batch(self): - data = self._get_data() - - if self.with_replacement: - idxs = self._sample_with_replacement(self.step) - else: - idxs = self._sample_without_replacement(self.step) - self.step += 1 - - xy = np.stack([data[i : i + self.sequence_length + 1] for i in idxs]).astype( - np.int64 - ) - x = torch.from_numpy(xy[:, :-1]).contiguous() - y = torch.from_numpy(xy[:, 1:]).contiguous() - return x, y - - def _sample_with_replacement(self, idx): - # Return an array of token indices of length self.batch_size - # Sampled with replacement, can get repeats at any time - seed = self.seed + idx * self.world_size + self.rank - rng = np.random.default_rng(seed) - return rng.integers(len(self), self.batch_size) - - def _shuffle_epoch(self, epoch): - seed = self.seed + epoch - rng = np.random.default_rng(seed) - # Drop one sequence to allow different offsets per epoch: - self.order = rng.permutation((len(self)) // self.sequence_length - 1) - # Shift all sequences in this epoch by this amount: - self.epoch_offset = rng.integers(self.sequence_length) - self.last_epoch = epoch - self.num_batches_of_seqlen = ( - len(self.order) // self.batch_size - ) # Drops remainder batch - - def _sample_without_replacement(self, step): - # Return an array of token indices of length self.batch_size - # Sampled without replacement, cycle all sequences before potential repeats - # Sequences are randomly offset in every epoch as well - batch_idx = self.world_size * step + self.rank - epoch_length = self.num_batches_of_seqlen - - epoch = batch_idx // epoch_length - if epoch != self.last_epoch: - self._shuffle_epoch(epoch) - epoch_idx = batch_idx % epoch_length - - start = epoch_idx * self.batch_size - end = start + self.batch_size - return self.order[start:end] * self.sequence_length + self.epoch_offset - - def num_batches(self): - if self.with_replacement: - return self.num_tokens // self.batch_size - return self.num_batches_of_seqlen diff --git a/src/experiment_budget.py b/src/experiment_budget.py new file mode 100644 index 0000000..a20797f --- /dev/null +++ b/src/experiment_budget.py @@ -0,0 +1,89 @@ +from dataclasses import asdict, dataclass +from typing import Iterable, Optional + + +@dataclass(frozen=True) +class TokenBudgetPlan: + train_token_budget: int + tokens_per_iteration: int + iterations: int + eval_at_tokens: tuple + eval_at_steps: tuple + data_unique_tokens: int + target_data_exposure: float + + def to_dict(self): + return asdict(self) + + +def _positive_int(value, name): + if value is None or value <= 0: + raise ValueError(f"{name} must be a positive integer, got {value}.") + return int(value) + + +def _tokens_to_steps(tokens, tokens_per_iteration, name): + if tokens % tokens_per_iteration != 0: + raise ValueError( + f"{name}={tokens} must be divisible by tokens_per_iteration=" + f"{tokens_per_iteration}. Choose an exact optimizer-step boundary." + ) + return tokens // tokens_per_iteration + + +def make_token_budget_plan( + *, + train_token_budget: int, + tokens_per_iteration: int, + data_unique_tokens: int, + eval_at_tokens: Optional[Iterable[int]] = None, + strict_sub_one_pass: bool = False, +) -> TokenBudgetPlan: + """Convert token budgets to exact optimizer-step boundaries.""" + + train_token_budget = _positive_int(train_token_budget, "train_token_budget") + tokens_per_iteration = _positive_int(tokens_per_iteration, "tokens_per_iteration") + data_unique_tokens = _positive_int(data_unique_tokens, "data_unique_tokens") + + if strict_sub_one_pass and train_token_budget > data_unique_tokens: + raise ValueError( + "strict_sub_one_pass requires train_token_budget <= " + f"data_unique_tokens, got {train_token_budget} > {data_unique_tokens}." + ) + + iterations = _tokens_to_steps( + train_token_budget, tokens_per_iteration, "train_token_budget" + ) + + requested_eval_tokens = ( + tuple(eval_at_tokens) if eval_at_tokens is not None else (train_token_budget,) + ) + if not requested_eval_tokens: + requested_eval_tokens = (train_token_budget,) + + normalized_eval_tokens = [] + for token_count in requested_eval_tokens: + token_count = _positive_int(token_count, "eval_at_tokens") + if token_count > train_token_budget: + raise ValueError( + f"eval_at_tokens contains {token_count}, which exceeds " + f"train_token_budget={train_token_budget}." + ) + normalized_eval_tokens.append(token_count) + + normalized_eval_tokens.append(train_token_budget) + normalized_eval_tokens = tuple(sorted(set(normalized_eval_tokens))) + eval_at_steps = tuple( + _tokens_to_steps(token_count, tokens_per_iteration, "eval_at_tokens") + for token_count in normalized_eval_tokens + ) + + return TokenBudgetPlan( + train_token_budget=train_token_budget, + tokens_per_iteration=tokens_per_iteration, + iterations=iterations, + eval_at_tokens=normalized_eval_tokens, + eval_at_steps=eval_at_steps, + data_unique_tokens=data_unique_tokens, + target_data_exposure=train_token_budget / data_unique_tokens, + ) diff --git a/src/main.py b/src/main.py index 215db42..3959fcf 100755 --- a/src/main.py +++ b/src/main.py @@ -14,17 +14,27 @@ import config import distributed from data.utils import DataReader, get_dataset +from experiment_budget import make_token_budget_plan from models.utils import get_model from optim.adafactor import Adafactor from optim.ademamix import AdEMAMix from optim.adopt import ADOPT from optim.base import train +from optim.composite import ( + CompositeOptimizer, + CompositeScheduler, + split_muon_param_groups, +) from optim.lamb import Lamb from optim.lion import Lion from optim.mars import MARS from optim.muon import CombinedScheduler, DistributedMuon, Muon from optim.prodigy import Prodigy -from optim.schedule import cos_inf_schedule, wsd_schedule +from optim.schedule import ( + cos_inf_schedule, + warmup_constant_schedule, + wsd_schedule, +) from optim.schedulefree import AdamWScheduleFree, SGDScheduleFree from optim.scion import Scion, ScionLight, scion_partitions from optim.sign import Signum @@ -85,6 +95,7 @@ def main(args, parser): print(f"Loading dataset: '{args.dataset}'") datareaders = get_data_readers(args) + configure_token_budget(args, datareaders["train"]) model = get_model(args).to( args.device @@ -327,9 +338,20 @@ def main(args, parser): momentum=args.momentum, ) elif args.opt == "muon-pytorch": - opt = torch.optim.Muon( - group_specs, + if not hasattr(torch.optim, "Muon"): + raise RuntimeError("muon-pytorch requires torch.optim.Muon (PyTorch >= 2.9).") + raw_model = distributed_backend.get_raw_model(model) + fallback_param_ids = { + id(raw_model.transformer.wte.weight), + id(raw_model.lm_head.weight), + } + muon_groups, adamw_groups = split_muon_param_groups( + group_specs, fallback_param_ids + ) + muon_opt = torch.optim.Muon( + muon_groups, lr=args.lr, + weight_decay=args.weight_decay, momentum=args.momentum, nesterov=args.nesterov, ns_steps=args.muon_ns_steps, @@ -341,6 +363,13 @@ def main(args, parser): eps=1e-7, # muon pytorch uses smaller eps adjust_lr_fn=None, # to make the orthogonalized update have a consistent RMS across rectangular matrices ) + adamw_opt = torch.optim.AdamW( + adamw_groups, + lr=args.muon_adamw_lr, + betas=(args.beta1, args.beta2), + weight_decay=args.weight_decay, + ) + opt = CompositeOptimizer([muon_opt, adamw_opt]) else: opt = torch.optim.SGD( group_specs, @@ -351,6 +380,14 @@ def main(args, parser): ) print(f"\nOptimizer:\n{opt}") + if isinstance(opt, CompositeOptimizer) and args.scheduler not in [ + "warmup_constant", + "none", + ]: + raise ValueError( + "muon-pytorch currently supports scheduler='warmup_constant' or 'none'." + ) + if args.scheduler != "none": assert ( args.warmup_steps < args.iterations @@ -402,6 +439,22 @@ def main(args, parser): if args.opt != "muon" else CombinedScheduler(opt, args) ) + elif args.scheduler == "warmup_constant": + lambda_schedule = warmup_constant_schedule( + n_warmup=args.warmup_steps, + init_div_factor=1e2, + ) + if isinstance(opt, CompositeOptimizer): + scheduler = CompositeScheduler( + [ + torch.optim.lr_scheduler.LambdaLR( + child_optimizer, lambda_schedule + ) + for child_optimizer in opt.optimizers + ] + ) + else: + scheduler = torch.optim.lr_scheduler.LambdaLR(opt, lambda_schedule) else: raise NotImplementedError(f"Unknown scheduler type: {args.scheduler}.") else: @@ -447,6 +500,8 @@ def get_data_readers(args, verbose=True): with_replacement=False, auto_shard=True, keep_in_ram=args.data_in_ram, + fixed_data_boundaries=args.fixed_data_boundaries, + lazy_data_permutation=args.lazy_data_permutation, ) val_reader = DataReader( data_src=data_srcs["val"], @@ -456,6 +511,8 @@ def get_data_readers(args, verbose=True): with_replacement=False, auto_shard=False, # NOTE Identical Per Rank keep_in_ram=args.data_in_ram, + fixed_data_boundaries=args.fixed_data_boundaries, + lazy_data_permutation=args.lazy_data_permutation, ) if verbose: @@ -468,6 +525,52 @@ def get_data_readers(args, verbose=True): } +def configure_token_budget(args, train_reader): + """Derive exact step/evaluation boundaries for token-budget experiments.""" + args.tokens_per_iteration = ( + args.world_size * args.batch_size * args.acc_steps * args.sequence_length + ) + args.data_unique_tokens = train_reader.unique_tokens_per_epoch + args.eval_at_steps = [] + args.token_budget_plan = None + + if args.train_token_budget is None: + if args.eval_at_tokens is not None: + raise ValueError( + "--eval_at_tokens requires --train_token_budget so step boundaries " + "can be derived unambiguously." + ) + return + + if args.strict_sub_one_pass and args.world_size != 1: + raise ValueError( + "--strict_sub_one_pass currently supports one GPU only. " + "Use the single-device backend." + ) + + plan = make_token_budget_plan( + train_token_budget=args.train_token_budget, + tokens_per_iteration=args.tokens_per_iteration, + data_unique_tokens=args.data_unique_tokens, + eval_at_tokens=args.eval_at_tokens, + strict_sub_one_pass=args.strict_sub_one_pass, + ) + args.iterations = plan.iterations + args.eval_at_steps = list(plan.eval_at_steps) + args.token_budget_plan = plan.to_dict() + + print( + "Token budget: " + f"{plan.train_token_budget:,} tokens, " + f"{plan.iterations:,} optimizer steps, " + f"{plan.target_data_exposure:.6f} effective passes" + ) + print( + "Evaluation token boundaries: " + + ", ".join(f"{value:,}" for value in plan.eval_at_tokens) + ) + + def get_exp_name( args, parser, @@ -488,7 +591,6 @@ def get_exp_name( "results_base_folder", "run_prefix", "wandb_run_prefix", - "seed", "device", "adema_beta3_warmup", "adema_alpha_warmup", @@ -510,6 +612,15 @@ def get_exp_name( "log_dynamics", "dynamics_logger_cfg", "experiment_name", + "eval_at_tokens", + "strict_sub_one_pass", + "fixed_data_boundaries", + "lazy_data_permutation", + "limit_final_eval", + "tokens_per_iteration", + "data_unique_tokens", + "eval_at_steps", + "token_budget_plan", ], ): # Set the custom exp name if needed diff --git a/src/optim/base.py b/src/optim/base.py index 72141ba..4149465 100755 --- a/src/optim/base.py +++ b/src/optim/base.py @@ -59,6 +59,10 @@ def train( load_worker_state(ckpt_dir) else: curr_iter = 0 + initial_iter = curr_iter + total_train_time_seconds = 0.0 + if "cuda" in cfg.device: + torch.cuda.reset_peak_memory_stats(torch.device(cfg.device)) if cfg.weight_average: # This does generally not support resuming training, but will work if @@ -101,7 +105,14 @@ def train( substep = curr_iter * cfg.acc_steps train_reader, val_reader = datareaders["train"], datareaders["val"] train_reader.set_step(substep) - stats = {"train_loss": [], "val_loss": [], "val_pp": [], "val_acc": []} + stats = { + "train_loss": [], + "val_loss": [], + "val_pp": [], + "val_acc": [], + "eval_history": [], + "iteration_time_seconds": [], + } grad_norms = [] model.train() @@ -126,11 +137,12 @@ def train( tokens = ws * substep * cfg.sequence_length * cfg.batch_size epoch = tokens / train_reader.num_tokens if ( - curr_iter % cfg.eval_interval == 0 + (cfg.eval_interval > 0 and curr_iter % cfg.eval_interval == 0) or curr_iter == cfg.iterations or (curr_iter in cfg.full_eval_at) + or (curr_iter in cfg.eval_at_steps) ): - eval_and_log( + eval_result = eval_and_log( tokens, curr_iter, epoch, @@ -142,6 +154,11 @@ def train( opt, full_eval=(curr_iter in cfg.full_eval_at), ) + if eval_result is not None: + stats["val_loss"].append(eval_result["val_loss"]) + stats["val_pp"].append(eval_result["val_perplexity"]) + stats["val_acc"].append(eval_result["val_accuracy"]) + stats["eval_history"].append(eval_result) if curr_iter > cfg.wa_interval and cfg.weight_average: eval_wa( @@ -238,8 +255,12 @@ def train( ewa.step(not_compiled_model, distributed_backend.is_master_process()) dt = (time.perf_counter_ns() - t_start) / 1e9 + total_train_time_seconds += dt curr_iter += 1 + tokens = ws * substep * cfg.sequence_length * cfg.batch_size + epoch = tokens / train_reader.num_tokens + data_exposure = tokens / cfg.data_unique_tokens if ( cfg.log_interval @@ -257,16 +278,19 @@ def train( prodigy_efective_lrs = log_prodigy_lr(opt) print( - f"Train: Iter={curr_iter} ({epoch:0.3f} epochs) " + f"Train: Iter={curr_iter} ({data_exposure:0.6f} effective passes) " f"train_loss={train_loss:.3f} iter_dt={dt:.2e}s " f"lr={current_lrs[0]:.2e}" ) + stats["train_loss"].append(train_loss) + stats["iteration_time_seconds"].append(dt) if cfg.opt == "prodigy": print(f"effective_lr={prodigy_efective_lrs[0]:.2e}") if cfg.wandb: wandb_logs = { "tokens": tokens, + "data_exposure": data_exposure, "iter": curr_iter, "train/loss": train_loss, "train/perplexity": 2.71828**train_loss, @@ -291,6 +315,25 @@ def train( grad_norms = [] + processed_tokens = ( + (cfg.iterations - initial_iter) + * ws + * cfg.acc_steps + * cfg.sequence_length + * cfg.batch_size + ) + stats["train_time_seconds"] = total_train_time_seconds + stats["processed_tokens_this_run"] = processed_tokens + stats["mean_tokens_per_second"] = ( + processed_tokens / total_train_time_seconds + if total_train_time_seconds > 0 + else None + ) + stats["peak_memory_bytes"] = ( + torch.cuda.max_memory_allocated(torch.device(cfg.device)) + if "cuda" in cfg.device + else None + ) return stats @@ -314,7 +357,7 @@ def eval_and_log( if cfg.opt == "sf-sgd" or cfg.opt == "sf-adamw": opt.eval() - if curr_iter == cfg.iterations or full_eval: + if (curr_iter == cfg.iterations and not cfg.limit_final_eval) or full_eval: max_num_batches = val_reader.num_batches() else: max_num_batches = cfg.eval_batches @@ -334,7 +377,8 @@ def eval_and_log( ) print( - f">Eval: Iter={curr_iter} ({epoch:0.3f} epochs) " + f">Eval: Iter={curr_iter} " + f"({tokens / cfg.data_unique_tokens:0.6f} effective passes) " f"val_loss={val_loss:.3f} " f"val_pp={val_perplexity:.3f} " f"val_acc={val_acc:3f}" @@ -344,6 +388,7 @@ def eval_and_log( if curr_iter == cfg.iterations or full_eval: logs = { "tokens": tokens, + "data_exposure": tokens / cfg.data_unique_tokens, "iter": curr_iter, "final-val/loss": val_loss, "final-val/perplexity": val_perplexity, @@ -353,6 +398,7 @@ def eval_and_log( else: logs = { "tokens": tokens, + "data_exposure": tokens / cfg.data_unique_tokens, "iter": curr_iter, "val/loss": val_loss, "val/perplexity": val_perplexity, @@ -378,4 +424,15 @@ def eval_and_log( text_table.add_data(curr_iter, val_perplexity, out_str) # why a copy? see github.com/wandb/wandb/issues/2981 wandb.log({f"generated-text-{wandb.run.name}": copy.copy(text_table)}) + result = { + "iter": curr_iter, + "tokens": tokens, + "data_exposure": tokens / cfg.data_unique_tokens, + "val_loss": val_loss, + "val_perplexity": val_perplexity, + "val_accuracy": val_acc, + "full_eval": full_eval, + "num_eval_batches": max_num_batches, + } model.train() + return result diff --git a/src/optim/composite.py b/src/optim/composite.py new file mode 100644 index 0000000..853eb2a --- /dev/null +++ b/src/optim/composite.py @@ -0,0 +1,79 @@ +class CompositeOptimizer: + """Minimal optimizer facade for algorithms that use two PyTorch optimizers.""" + + def __init__(self, optimizers): + self.optimizers = list(optimizers) + if not self.optimizers: + raise ValueError("CompositeOptimizer requires at least one optimizer.") + self.param_groups = [ + group for optimizer in self.optimizers for group in optimizer.param_groups + ] + + def step(self): + for optimizer in self.optimizers: + optimizer.step() + + def zero_grad(self, *args, **kwargs): + for optimizer in self.optimizers: + optimizer.zero_grad(*args, **kwargs) + + def state_dict(self): + return {"optimizers": [optimizer.state_dict() for optimizer in self.optimizers]} + + def load_state_dict(self, state_dict): + states = state_dict["optimizers"] + if len(states) != len(self.optimizers): + raise ValueError( + "Checkpoint optimizer count does not match CompositeOptimizer." + ) + for optimizer, optimizer_state in zip(self.optimizers, states): + optimizer.load_state_dict(optimizer_state) + + +class CompositeScheduler: + def __init__(self, schedulers): + self.schedulers = list(schedulers) + + def step(self): + for scheduler in self.schedulers: + scheduler.step() + + def state_dict(self): + return {"schedulers": [scheduler.state_dict() for scheduler in self.schedulers]} + + def load_state_dict(self, state_dict): + states = state_dict["schedulers"] + if len(states) != len(self.schedulers): + raise ValueError( + "Checkpoint scheduler count does not match CompositeScheduler." + ) + for scheduler, scheduler_state in zip(self.schedulers, states): + scheduler.load_state_dict(scheduler_state) + + +def split_muon_param_groups(group_specs, fallback_param_ids): + """Partition groups between PyTorch Muon and an AdamW fallback.""" + muon_groups = [] + adamw_groups = [] + for group in group_specs: + shared = {key: value for key, value in group.items() if key != "params"} + muon_params = [ + parameter + for parameter in group["params"] + if parameter.ndim == 2 and id(parameter) not in fallback_param_ids + ] + adamw_params = [ + parameter + for parameter in group["params"] + if parameter.ndim != 2 or id(parameter) in fallback_param_ids + ] + if muon_params: + muon_groups.append({**shared, "params": muon_params}) + if adamw_params: + adamw_groups.append({**shared, "params": adamw_params}) + if not muon_groups or not adamw_groups: + raise ValueError( + "muon-pytorch requires both hidden 2D matrices and AdamW fallback " + "parameters. Check the model parameter partition." + ) + return muon_groups, adamw_groups diff --git a/src/optim/schedule.py b/src/optim/schedule.py index 267c6c8..57c99a2 100644 --- a/src/optim/schedule.py +++ b/src/optim/schedule.py @@ -3,6 +3,24 @@ import numpy as np +def warmup_constant_schedule(n_warmup, init_div_factor=100): + """Linear warmup followed by a horizon-independent constant learning rate.""" + if n_warmup < 0: + raise ValueError(f"n_warmup must be non-negative, got {n_warmup}.") + if init_div_factor <= 0: + raise ValueError( + f"init_div_factor must be positive, got {init_div_factor}." + ) + + def schedule(step): + if n_warmup == 0 or step >= n_warmup: + return 1.0 + progress = step / n_warmup + return progress + (1 - progress) / init_div_factor + + return schedule + + def cos_inf_schedule(n_iterations, n_warmup, div_factor, final_div_factor, n_inf): """Cosine annealing with warmup and _constant_ final_lr after cycle ended. Args: diff --git a/tests/test_composite_optimizer.py b/tests/test_composite_optimizer.py new file mode 100644 index 0000000..2b1903c --- /dev/null +++ b/tests/test_composite_optimizer.py @@ -0,0 +1,55 @@ +import sys +import unittest +from pathlib import Path + +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from optim.composite import ( + CompositeOptimizer, + CompositeScheduler, + split_muon_param_groups, +) + + +class CompositeOptimizerTest(unittest.TestCase): + def test_steps_and_restores_both_optimizers(self): + first = torch.nn.Parameter(torch.tensor([1.0])) + second = torch.nn.Parameter(torch.tensor([2.0])) + first_opt = torch.optim.SGD([first], lr=0.1) + second_opt = torch.optim.AdamW([second], lr=0.01) + optimizer = CompositeOptimizer([first_opt, second_opt]) + scheduler = CompositeScheduler( + [ + torch.optim.lr_scheduler.LambdaLR(first_opt, lambda _: 1.0), + torch.optim.lr_scheduler.LambdaLR(second_opt, lambda _: 1.0), + ] + ) + + first.grad = torch.ones_like(first) + second.grad = torch.ones_like(second) + optimizer.step() + scheduler.step() + + self.assertLess(first.item(), 1.0) + self.assertLess(second.item(), 2.0) + self.assertEqual(len(optimizer.param_groups), 2) + + optimizer.load_state_dict(optimizer.state_dict()) + scheduler.load_state_dict(scheduler.state_dict()) + + def test_partitions_embeddings_and_vectors_to_adamw(self): + hidden = torch.nn.Parameter(torch.ones(4, 4)) + embedding = torch.nn.Parameter(torch.ones(8, 4)) + norm = torch.nn.Parameter(torch.ones(4)) + groups = [{"params": [hidden, embedding, norm], "weight_decay": 0.1}] + + muon_groups, adamw_groups = split_muon_param_groups(groups, {id(embedding)}) + + self.assertEqual(muon_groups[0]["params"], [hidden]) + self.assertEqual(adamw_groups[0]["params"], [embedding, norm]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_data_reader.py b/tests/test_data_reader.py new file mode 100644 index 0000000..3dda444 --- /dev/null +++ b/tests/test_data_reader.py @@ -0,0 +1,42 @@ +import sys +import unittest +from pathlib import Path + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + + +class DataReaderTest(unittest.TestCase): + def test_fixed_boundaries_and_no_replacement(self): + from data.reader import DataReader + + sequence_length = 8 + batch_size = 2 + data = np.arange(8 * 13 + 1, dtype=np.uint16) + reader = DataReader( + data, + batch_size=batch_size, + sequence_length=sequence_length, + seed=7, + fixed_data_boundaries=True, + lazy_data_permutation=True, + ) + + starts = [] + for _ in range(reader.num_batches()): + x, _ = reader.sample_batch() + starts.extend(x[:, 0].tolist()) + + self.assertEqual(reader.epoch_offset, 0) + self.assertEqual(len(starts), len(set(starts))) + self.assertIsNone(reader.order) + self.assertTrue(all(value % sequence_length == 0 for value in starts)) + self.assertEqual( + reader.unique_tokens_per_epoch, + reader.num_batches() * batch_size * sequence_length, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_experiment_budget.py b/tests/test_experiment_budget.py new file mode 100644 index 0000000..3d28beb --- /dev/null +++ b/tests/test_experiment_budget.py @@ -0,0 +1,50 @@ +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from experiment_budget import make_token_budget_plan + + +class TokenBudgetPlanTest(unittest.TestCase): + def test_builds_exact_nested_plan(self): + plan = make_token_budget_plan( + train_token_budget=268_435_456, + tokens_per_iteration=65_536, + data_unique_tokens=1_000_013_824, + eval_at_tokens=[ + 16_777_216, + 33_554_432, + 67_108_864, + 134_217_728, + 268_435_456, + ], + strict_sub_one_pass=True, + ) + + self.assertEqual(plan.iterations, 4096) + self.assertEqual(plan.eval_at_steps, (256, 512, 1024, 2048, 4096)) + self.assertLess(plan.target_data_exposure, 1.0) + + def test_rejects_inexact_step_boundary(self): + with self.assertRaisesRegex(ValueError, "must be divisible"): + make_token_budget_plan( + train_token_budget=1_000_000, + tokens_per_iteration=65_536, + data_unique_tokens=2_000_000, + strict_sub_one_pass=True, + ) + + def test_rejects_more_than_one_strict_pass(self): + with self.assertRaisesRegex(ValueError, "strict_sub_one_pass"): + make_token_budget_plan( + train_token_budget=131_072, + tokens_per_iteration=65_536, + data_unique_tokens=65_536, + strict_sub_one_pass=True, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_fineweb_stream.py b/tests/test_fineweb_stream.py new file mode 100644 index 0000000..bdc1fe5 --- /dev/null +++ b/tests/test_fineweb_stream.py @@ -0,0 +1,50 @@ +import importlib.util +import unittest +from pathlib import Path + +import numpy as np + +SCRIPT_PATH = ( + Path(__file__).resolve().parents[1] + / "scripts" + / "sub_one_pass" + / "prepare_fineweb_stream.py" +) +SPEC = importlib.util.spec_from_file_location("prepare_fineweb_stream", SCRIPT_PATH) +PREPARE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(PREPARE) + + +class FakeTokenizer: + eot_token = 99 + + def encode_ordinary(self, text): + return { + "validation-one": [1, 2, 3, 4], + "boundary-document": [5, 6, 7, 8], + "training-one": [9, 10, 11], + }[text] + + +class FineWebStreamTest(unittest.TestCase): + def test_does_not_split_one_document_between_validation_and_train(self): + stream = [ + {"text": "validation-one"}, + {"text": "boundary-document"}, + {"text": "training-one"}, + ] + validation = np.zeros(7, dtype=np.uint16) + train = np.zeros(4, dtype=np.uint16) + + documents, discarded = PREPARE.write_tokens( + stream, FakeTokenizer(), validation, train + ) + + self.assertEqual(documents, 3) + self.assertEqual(discarded, 3) + self.assertEqual(validation.tolist(), [1, 2, 3, 4, 99, 5, 6]) + self.assertEqual(train.tolist(), [9, 10, 11, 99]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_schedule.py b/tests/test_schedule.py new file mode 100644 index 0000000..116e09f --- /dev/null +++ b/tests/test_schedule.py @@ -0,0 +1,24 @@ +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from optim.schedule import warmup_constant_schedule + + +class WarmupConstantScheduleTest(unittest.TestCase): + def test_warms_up_then_stays_constant(self): + schedule = warmup_constant_schedule(n_warmup=4, init_div_factor=100) + self.assertAlmostEqual(schedule(0), 0.01) + self.assertLess(schedule(1), schedule(2)) + self.assertEqual(schedule(4), 1.0) + self.assertEqual(schedule(400), 1.0) + + def test_zero_warmup(self): + schedule = warmup_constant_schedule(n_warmup=0) + self.assertEqual(schedule(0), 1.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sub_one_pass_runner.py b/tests/test_sub_one_pass_runner.py new file mode 100644 index 0000000..4eb9500 --- /dev/null +++ b/tests/test_sub_one_pass_runner.py @@ -0,0 +1,112 @@ +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +RUNNER_PATH = ( + Path(__file__).resolve().parents[1] / "scripts" / "sub_one_pass" / "run_all.py" +) +SPEC = importlib.util.spec_from_file_location("sub_one_pass_run_all", RUNNER_PATH) +RUNNER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(RUNNER) + + +class SubOnePassRunnerTest(unittest.TestCase): + def test_registry_matches_repository_optimizer_surface(self): + expected = { + "adamw", + "sgd", + "muon", + "soap", + "ademamix", + "lion", + "sf-adamw", + "sf-sgd", + "signsgd", + "signum", + "prodigy", + "sophiag", + "adopt", + "mars", + "adafactor", + "lamb", + "scion", + "scion-light", + "d-muon", + "muon-pytorch", + } + self.assertEqual(set(RUNNER.OPTIMIZER_ARGS), expected) + self.assertEqual(set(RUNNER.LR_FLAGS), expected) + + def test_every_optimizer_configuration_is_flag_value_pairs(self): + for optimizer, arguments in RUNNER.OPTIMIZER_ARGS.items(): + with self.subTest(optimizer=optimizer): + self.assertEqual(len(arguments) % 2, 0) + self.assertTrue(all(flag.startswith("--") for flag in arguments[::2])) + + def test_scales_all_optimizer_learning_rates(self): + for optimizer in RUNNER.OPTIMIZER_ARGS: + with self.subTest(optimizer=optimizer): + original = RUNNER.scaled_optimizer_args(optimizer, 1.0) + scaled = RUNNER.scaled_optimizer_args(optimizer, 3.0) + for index, flag in enumerate(original[::2]): + value_index = index * 2 + 1 + if flag in RUNNER.LR_FLAGS[optimizer]: + self.assertAlmostEqual( + float(scaled[value_index]), + 3.0 * float(original[value_index]), + ) + else: + self.assertEqual(scaled[value_index], original[value_index]) + + def test_stage_budgets_are_exact_step_boundaries(self): + tokens_per_step = 16 * 8 * 512 + for stage, boundaries in RUNNER.STAGE_EVAL_TOKENS.items(): + with self.subTest(stage=stage): + self.assertEqual(boundaries, sorted(set(boundaries))) + self.assertTrue( + all(tokens % tokens_per_step == 0 for tokens in boundaries) + ) + + def test_manifest_preserves_runs_across_invocations(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "manifest_tune.json" + args = SimpleNamespace(stage="tune", marker="first") + manifest = RUNNER.initialize_manifest(path, args, ["adamw"]) + RUNNER.upsert_run_record( + manifest, + { + "experiment_dir": "/tmp/adamw-scale1", + "optimizer": "adamw", + "status": "completed", + }, + ) + RUNNER.write_manifest(path, manifest) + + args = SimpleNamespace(stage="tune", marker="second") + restored = RUNNER.initialize_manifest(path, args, ["soap"]) + self.assertEqual(restored["optimizers"], ["adamw", "soap"]) + self.assertEqual(len(restored["invocations"]), 2) + self.assertEqual(len(restored["runs"]), 1) + + record = RUNNER.upsert_run_record( + restored, + { + "experiment_dir": "/tmp/adamw-scale1", + "optimizer": "adamw", + "status": "skipped-completed", + }, + ) + self.assertEqual(len(restored["runs"]), 1) + self.assertEqual(record["status"], "skipped-completed") + RUNNER.write_manifest(path, restored) + self.assertEqual( + json.loads(path.read_text())["runs"][0]["status"], + "skipped-completed", + ) + + +if __name__ == "__main__": + unittest.main()