Add files using upload-large-folder tool

config.json

@@ -0,0 +1,38 @@
+{
+  "_name_or_path": "BAAI/Emu3-Stage1",
+  "architectures": [
+    "Emu3ForCausalLM"
+  ],
+  "attention_dropout": 0.1,
+  "auto_map": {
+    "AutoConfig": "BAAI/Emu3-Stage1--configuration_emu3.Emu3Config",
+    "AutoModelForCausalLM": "BAAI/Emu3-Stage1--modeling_emu3.Emu3ForCausalLM"
+  },
+  "boi_token_id": 151852,
+  "bos_token_id": 151849,
+  "eof_token_id": 151847,
+  "eoi_token_id": 151853,
+  "eol_token_id": 151846,
+  "eos_token_id": 151850,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "image_area": 65536,
+  "img_token_id": 151851,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 4300,
+  "model_type": "Emu3",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pad_token_id": 151643,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.0",
+  "use_cache": false,
+  "vocab_size": 184622
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 151849,
+  "eos_token_id": 151850,
+  "pad_token_id": 151643,
+  "transformers_version": "4.44.0"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b62bd5a0f728b907d5434122b0b00105227562b67dde9947c89a45a20131ff08
+size 4884766656

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad4f26d4327d067f655bbd01379c055c371801ba7b162d696c40dff8395a60a0
+size 4999819320

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89b90243d09e9dcd8bd9a66a956655daa84090dde7350419e46667edbf3add05
+size 4915916184

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8ae530aa5a8f55aa19a2323303b08ae793dcf4bf8b344ea45b195b7a08b60cb
+size 2183554760

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 16984023040
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:831eb06136e85ca16b0334198bc5832e10133705ea33d6bed6a259c1d26ada79
+size 15984

rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9c825d8b3fd06672abece05a6b769b5f8eed2e2936452729252a51f5c2dcece
+size 15984

rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bba2855245af9e7ec65925d5af0c86931f57023265289ce75d38acd370a3c443
+size 15984

rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38d70022eaf19dc87b3d8743cf1186a9d9243656c6c970bb411c881221cee52f
+size 15984

rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c63969e02eb647b7f2dc990ed91700ba9b97298401efa2ae7f1b801dc271d8c
+size 15984

rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa75218af660315f24bff8d48e07714561ad2c9967d3d7df489dd9715ca2f21c
+size 15984

rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f53e371627d8b0ac7b341be0c9d8896b642f720e5a4efdd589489ab8bde1a5c2
+size 15984

rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ff4395d1ff0cd6739686f27e4217dc02163942adc431ea1b9d1e6c577f45a03
+size 15984

trainer_state.json

@@ -0,0 +1,1825 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.204262623372908,
+  "eval_steps": 400,
+  "global_step": 11200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.014304105278214848,
+      "grad_norm": 3.5871307485399613,
+      "learning_rate": 1.6666666666666667e-06,
+      "loss": 4.2171,
+      "step": 50
+    },
+    {
+      "epoch": 0.028608210556429696,
+      "grad_norm": 8.578020193760794,
+      "learning_rate": 3.3333333333333333e-06,
+      "loss": 3.5086,
+      "step": 100
+    },
+    {
+      "epoch": 0.04291231583464454,
+      "grad_norm": 3.453360153921811,
+      "learning_rate": 5e-06,
+      "loss": 2.7411,
+      "step": 150
+    },
+    {
+      "epoch": 0.05721642111285939,
+      "grad_norm": 6.637419959801433,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 2.5587,
+      "step": 200
+    },
+    {
+      "epoch": 0.07152052639107424,
+      "grad_norm": 5.39103050730964,
+      "learning_rate": 8.333333333333334e-06,
+      "loss": 2.4626,
+      "step": 250
+    },
+    {
+      "epoch": 0.08582463166928908,
+      "grad_norm": 10.145361416938202,
+      "learning_rate": 1e-05,
+      "loss": 2.3934,
+      "step": 300
+    },
+    {
+      "epoch": 0.10012873694750393,
+      "grad_norm": 7.783722807777708,
+      "learning_rate": 9.999953760295448e-06,
+      "loss": 2.272,
+      "step": 350
+    },
+    {
+      "epoch": 0.11443284222571878,
+      "grad_norm": 3.65791345167478,
+      "learning_rate": 9.999815042132062e-06,
+      "loss": 2.2033,
+      "step": 400
+    },
+    {
+      "epoch": 0.11443284222571878,
+      "eval_loss": 2.1356396675109863,
+      "eval_runtime": 13.2089,
+      "eval_samples_per_second": 75.707,
+      "eval_steps_per_second": 2.423,
+      "step": 400
+    },
+    {
+      "epoch": 0.12873694750393364,
+      "grad_norm": 8.621408307551274,
+      "learning_rate": 9.999583848360633e-06,
+      "loss": 2.1457,
+      "step": 450
+    },
+    {
+      "epoch": 0.1430410527821485,
+      "grad_norm": 6.2605099034671,
+      "learning_rate": 9.999260183732424e-06,
+      "loss": 2.1022,
+      "step": 500
+    },
+    {
+      "epoch": 0.1573451580603633,
+      "grad_norm": 7.684157630671387,
+      "learning_rate": 9.998844054899058e-06,
+      "loss": 2.0793,
+      "step": 550
+    },
+    {
+      "epoch": 0.17164926333857816,
+      "grad_norm": 1.7771086513044794,
+      "learning_rate": 9.998335470412393e-06,
+      "loss": 2.0783,
+      "step": 600
+    },
+    {
+      "epoch": 0.185953368616793,
+      "grad_norm": 4.839674439593447,
+      "learning_rate": 9.997734440724333e-06,
+      "loss": 2.0455,
+      "step": 650
+    },
+    {
+      "epoch": 0.20025747389500786,
+      "grad_norm": 1.7547598316419108,
+      "learning_rate": 9.997040978186633e-06,
+      "loss": 2.0206,
+      "step": 700
+    },
+    {
+      "epoch": 0.21456157917322272,
+      "grad_norm": 2.5514452897242035,
+      "learning_rate": 9.996255097050624e-06,
+      "loss": 2.003,
+      "step": 750
+    },
+    {
+      "epoch": 0.22886568445143757,
+      "grad_norm": 3.9407907366389976,
+      "learning_rate": 9.995376813466934e-06,
+      "loss": 1.9948,
+      "step": 800
+    },
+    {
+      "epoch": 0.22886568445143757,
+      "eval_loss": 1.9659738540649414,
+      "eval_runtime": 13.202,
+      "eval_samples_per_second": 75.746,
+      "eval_steps_per_second": 2.424,
+      "step": 800
+    },
+    {
+      "epoch": 0.24316978972965242,
+      "grad_norm": 1.7301759518540019,
+      "learning_rate": 9.994406145485151e-06,
+      "loss": 1.9861,
+      "step": 850
+    },
+    {
+      "epoch": 0.25747389500786727,
+      "grad_norm": 4.606477014527872,
+      "learning_rate": 9.993343113053454e-06,
+      "loss": 1.9713,
+      "step": 900
+    },
+    {
+      "epoch": 0.2717780002860821,
+      "grad_norm": 2.8102943343276183,
+      "learning_rate": 9.992187738018203e-06,
+      "loss": 1.9675,
+      "step": 950
+    },
+    {
+      "epoch": 0.286082105564297,
+      "grad_norm": 1.6938293305410583,
+      "learning_rate": 9.99094004412348e-06,
+      "loss": 1.9556,
+      "step": 1000
+    },
+    {
+      "epoch": 0.3003862108425118,
+      "grad_norm": 3.4489466081512044,
+      "learning_rate": 9.989600057010625e-06,
+      "loss": 1.9417,
+      "step": 1050
+    },
+    {
+      "epoch": 0.3146903161207266,
+      "grad_norm": 1.454314969912362,
+      "learning_rate": 9.988167804217682e-06,
+      "loss": 1.9361,
+      "step": 1100
+    },
+    {
+      "epoch": 0.3289944213989415,
+      "grad_norm": 1.168052123243911,
+      "learning_rate": 9.986643315178848e-06,
+      "loss": 1.9264,
+      "step": 1150
+    },
+    {
+      "epoch": 0.3432985266771563,
+      "grad_norm": 1.7334339924315771,
+      "learning_rate": 9.98502662122387e-06,
+      "loss": 1.9308,
+      "step": 1200
+    },
+    {
+      "epoch": 0.3432985266771563,
+      "eval_loss": 1.8975478410720825,
+      "eval_runtime": 13.3147,
+      "eval_samples_per_second": 75.105,
+      "eval_steps_per_second": 2.403,
+      "step": 1200
+    },
+    {
+      "epoch": 0.3576026319553712,
+      "grad_norm": 1.5051784354611382,
+      "learning_rate": 9.983317755577392e-06,
+      "loss": 1.9208,
+      "step": 1250
+    },
+    {
+      "epoch": 0.371906737233586,
+      "grad_norm": 1.5926471600228091,
+      "learning_rate": 9.981516753358274e-06,
+      "loss": 1.9156,
+      "step": 1300
+    },
+    {
+      "epoch": 0.3862108425118009,
+      "grad_norm": 1.4680846710500992,
+      "learning_rate": 9.979623651578881e-06,
+      "loss": 1.9003,
+      "step": 1350
+    },
+    {
+      "epoch": 0.40051494779001573,
+      "grad_norm": 1.3763604778464051,
+      "learning_rate": 9.977638489144308e-06,
+      "loss": 1.9069,
+      "step": 1400
+    },
+    {
+      "epoch": 0.4148190530682306,
+      "grad_norm": 1.524436140637499,
+      "learning_rate": 9.975561306851585e-06,
+      "loss": 1.8973,
+      "step": 1450
+    },
+    {
+      "epoch": 0.42912315834644543,
+      "grad_norm": 1.6595586978206922,
+      "learning_rate": 9.973392147388847e-06,
+      "loss": 1.8994,
+      "step": 1500
+    },
+    {
+      "epoch": 0.4434272636246603,
+      "grad_norm": 1.5658370421245245,
+      "learning_rate": 9.971131055334445e-06,
+      "loss": 1.8875,
+      "step": 1550
+    },
+    {
+      "epoch": 0.45773136890287514,
+      "grad_norm": 1.9874689334717497,
+      "learning_rate": 9.968778077156035e-06,
+      "loss": 1.8885,
+      "step": 1600
+    },
+    {
+      "epoch": 0.45773136890287514,
+      "eval_loss": 1.8600050210952759,
+      "eval_runtime": 13.4365,
+      "eval_samples_per_second": 74.424,
+      "eval_steps_per_second": 2.382,
+      "step": 1600
+    },
+    {
+      "epoch": 0.47203547418109,
+      "grad_norm": 1.391686515937597,
+      "learning_rate": 9.966333261209625e-06,
+      "loss": 1.8825,
+      "step": 1650
+    },
+    {
+      "epoch": 0.48633957945930484,
+      "grad_norm": 1.250031324432951,
+      "learning_rate": 9.96379665773858e-06,
+      "loss": 1.8812,
+      "step": 1700
+    },
+    {
+      "epoch": 0.5006436847375196,
+      "grad_norm": 0.937500185481468,
+      "learning_rate": 9.961168318872583e-06,
+      "loss": 1.873,
+      "step": 1750
+    },
+    {
+      "epoch": 0.5149477900157345,
+      "grad_norm": 1.3831225145690145,
+      "learning_rate": 9.958448298626576e-06,
+      "loss": 1.8733,
+      "step": 1800
+    },
+    {
+      "epoch": 0.5292518952939493,
+      "grad_norm": 2.8853368812546845,
+      "learning_rate": 9.95563665289964e-06,
+      "loss": 1.8607,
+      "step": 1850
+    },
+    {
+      "epoch": 0.5435560005721642,
+      "grad_norm": 1.8333114897013831,
+      "learning_rate": 9.952733439473847e-06,
+      "loss": 1.867,
+      "step": 1900
+    },
+    {
+      "epoch": 0.557860105850379,
+      "grad_norm": 1.4955344425097794,
+      "learning_rate": 9.94973871801308e-06,
+      "loss": 1.8668,
+      "step": 1950
+    },
+    {
+      "epoch": 0.572164211128594,
+      "grad_norm": 1.058766324989062,
+      "learning_rate": 9.946652550061798e-06,
+      "loss": 1.8468,
+      "step": 2000
+    },
+    {
+      "epoch": 0.572164211128594,
+      "eval_loss": 1.8319525718688965,
+      "eval_runtime": 13.2128,
+      "eval_samples_per_second": 75.684,
+      "eval_steps_per_second": 2.422,
+      "step": 2000
+    },
+    {
+      "epoch": 0.5864683164068087,
+      "grad_norm": 1.061225891590699,
+      "learning_rate": 9.943474999043775e-06,
+      "loss": 1.8513,
+      "step": 2050
+    },
+    {
+      "epoch": 0.6007724216850236,
+      "grad_norm": 1.8307541279709998,
+      "learning_rate": 9.9402061302608e-06,
+      "loss": 1.8489,
+      "step": 2100
+    },
+    {
+      "epoch": 0.6150765269632384,
+      "grad_norm": 1.3493002065305932,
+      "learning_rate": 9.93684601089133e-06,
+      "loss": 1.8428,
+      "step": 2150
+    },
+    {
+      "epoch": 0.6293806322414532,
+      "grad_norm": 1.3269702867770274,
+      "learning_rate": 9.933394709989109e-06,
+      "loss": 1.8485,
+      "step": 2200
+    },
+    {
+      "epoch": 0.6436847375196681,
+      "grad_norm": 1.3587430620093466,
+      "learning_rate": 9.92985229848175e-06,
+      "loss": 1.8323,
+      "step": 2250
+    },
+    {
+      "epoch": 0.657988842797883,
+      "grad_norm": 1.6220456911234815,
+      "learning_rate": 9.926218849169284e-06,
+      "loss": 1.8417,
+      "step": 2300
+    },
+    {
+      "epoch": 0.6722929480760979,
+      "grad_norm": 1.3306733931984986,
+      "learning_rate": 9.922494436722653e-06,
+      "loss": 1.8319,
+      "step": 2350
+    },
+    {
+      "epoch": 0.6865970533543126,
+      "grad_norm": 1.841456096268586,
+      "learning_rate": 9.91867913768218e-06,
+      "loss": 1.8221,
+      "step": 2400
+    },
+    {
+      "epoch": 0.6865970533543126,
+      "eval_loss": 1.8098633289337158,
+      "eval_runtime": 13.2499,
+      "eval_samples_per_second": 75.472,
+      "eval_steps_per_second": 2.415,
+      "step": 2400
+    },
+    {
+      "epoch": 0.7009011586325276,
+      "grad_norm": 1.0743552491621962,
+      "learning_rate": 9.914773030456001e-06,
+      "loss": 1.8264,
+      "step": 2450
+    },
+    {
+      "epoch": 0.7152052639107424,
+      "grad_norm": 0.9817511549306134,
+      "learning_rate": 9.910776195318448e-06,
+      "loss": 1.8339,
+      "step": 2500
+    },
+    {
+      "epoch": 0.7295093691889573,
+      "grad_norm": 2.5586651091316166,
+      "learning_rate": 9.906688714408396e-06,
+      "loss": 1.8361,
+      "step": 2550
+    },
+    {
+      "epoch": 0.743813474467172,
+      "grad_norm": 1.3521713422261818,
+      "learning_rate": 9.902510671727583e-06,
+      "loss": 1.8172,
+      "step": 2600
+    },
+    {
+      "epoch": 0.758117579745387,
+      "grad_norm": 1.196264818533523,
+      "learning_rate": 9.898242153138882e-06,
+      "loss": 1.815,
+      "step": 2650
+    },
+    {
+      "epoch": 0.7724216850236018,
+      "grad_norm": 1.1236416672692373,
+      "learning_rate": 9.89388324636453e-06,
+      "loss": 1.8217,
+      "step": 2700
+    },
+    {
+      "epoch": 0.7867257903018167,
+      "grad_norm": 1.0482615125286014,
+      "learning_rate": 9.889434040984333e-06,
+      "loss": 1.8111,
+      "step": 2750
+    },
+    {
+      "epoch": 0.8010298955800315,
+      "grad_norm": 1.2877895138097581,
+      "learning_rate": 9.88489462843382e-06,
+      "loss": 1.8139,
+      "step": 2800
+    },
+    {
+      "epoch": 0.8010298955800315,
+      "eval_loss": 1.7917475700378418,
+      "eval_runtime": 13.2172,
+      "eval_samples_per_second": 75.659,
+      "eval_steps_per_second": 2.421,
+      "step": 2800
+    },
+    {
+      "epoch": 0.8153340008582464,
+      "grad_norm": 1.3263953998902482,
+      "learning_rate": 9.880265102002369e-06,
+      "loss": 1.7969,
+      "step": 2850
+    },
+    {
+      "epoch": 0.8296381061364612,
+      "grad_norm": 0.8011721107369111,
+      "learning_rate": 9.875545556831283e-06,
+      "loss": 1.8123,
+      "step": 2900
+    },
+    {
+      "epoch": 0.843942211414676,
+      "grad_norm": 1.452056398499391,
+      "learning_rate": 9.870736089911836e-06,
+      "loss": 1.8137,
+      "step": 2950
+    },
+    {
+      "epoch": 0.8582463166928909,
+      "grad_norm": 1.1618490357992515,
+      "learning_rate": 9.865836800083291e-06,
+      "loss": 1.812,
+      "step": 3000
+    },
+    {
+      "epoch": 0.8725504219711057,
+      "grad_norm": 1.2693408191050113,
+      "learning_rate": 9.860847788030852e-06,
+      "loss": 1.7989,
+      "step": 3050
+    },
+    {
+      "epoch": 0.8868545272493206,
+      "grad_norm": 1.1394593474503365,
+      "learning_rate": 9.855769156283604e-06,
+      "loss": 1.8068,
+      "step": 3100
+    },
+    {
+      "epoch": 0.9011586325275354,
+      "grad_norm": 1.739305216161417,
+      "learning_rate": 9.850601009212408e-06,
+      "loss": 1.8026,
+      "step": 3150
+    },
+    {
+      "epoch": 0.9154627378057503,
+      "grad_norm": 0.9509459350795362,
+      "learning_rate": 9.845343453027747e-06,
+      "loss": 1.8055,
+      "step": 3200
+    },
+    {
+      "epoch": 0.9154627378057503,
+      "eval_loss": 1.7801611423492432,
+      "eval_runtime": 13.218,
+      "eval_samples_per_second": 75.654,
+      "eval_steps_per_second": 2.421,
+      "step": 3200
+    },
+    {
+      "epoch": 0.9297668430839651,
+      "grad_norm": 1.279124709454881,
+      "learning_rate": 9.839996595777552e-06,
+      "loss": 1.7968,
+      "step": 3250
+    },
+    {
+      "epoch": 0.94407094836218,
+      "grad_norm": 1.092437156034456,
+      "learning_rate": 9.83456054734498e-06,
+      "loss": 1.7892,
+      "step": 3300
+    },
+    {
+      "epoch": 0.9583750536403948,
+      "grad_norm": 0.9619111877602855,
+      "learning_rate": 9.829035419446156e-06,
+      "loss": 1.7951,
+      "step": 3350
+    },
+    {
+      "epoch": 0.9726791589186097,
+      "grad_norm": 0.9476479377652314,
+      "learning_rate": 9.823421325627865e-06,
+      "loss": 1.8003,
+      "step": 3400
+    },
+    {
+      "epoch": 0.9869832641968245,
+      "grad_norm": 0.7761637732992411,
+      "learning_rate": 9.81771838126524e-06,
+      "loss": 1.7852,
+      "step": 3450
+    },
+    {
+      "epoch": 1.0012873694750393,
+      "grad_norm": 1.6427439742947594,
+      "learning_rate": 9.811926703559374e-06,
+      "loss": 1.7813,
+      "step": 3500
+    },
+    {
+      "epoch": 1.0155914747532542,
+      "grad_norm": 1.1995527718721077,
+      "learning_rate": 9.806046411534916e-06,
+      "loss": 1.7693,
+      "step": 3550
+    },
+    {
+      "epoch": 1.029895580031469,
+      "grad_norm": 0.6804399746581633,
+      "learning_rate": 9.800077626037633e-06,
+      "loss": 1.7779,
+      "step": 3600
+    },
+    {
+      "epoch": 1.029895580031469,
+      "eval_loss": 1.770551323890686,
+      "eval_runtime": 13.1907,
+      "eval_samples_per_second": 75.811,
+      "eval_steps_per_second": 2.426,
+      "step": 3600
+    },
+    {
+      "epoch": 1.044199685309684,
+      "grad_norm": 0.9739432549705861,
+      "learning_rate": 9.794020469731915e-06,
+      "loss": 1.7844,
+      "step": 3650
+    },
+    {
+      "epoch": 1.0585037905878987,
+      "grad_norm": 1.24072763349112,
+      "learning_rate": 9.787875067098257e-06,
+      "loss": 1.7826,
+      "step": 3700
+    },
+    {
+      "epoch": 1.0728078958661136,
+      "grad_norm": 1.0497808533387654,
+      "learning_rate": 9.781641544430703e-06,
+      "loss": 1.7875,
+      "step": 3750
+    },
+    {
+      "epoch": 1.0871120011443285,
+      "grad_norm": 1.608878092364201,
+      "learning_rate": 9.775320029834255e-06,
+      "loss": 1.7676,
+      "step": 3800
+    },
+    {
+      "epoch": 1.1014161064225432,
+      "grad_norm": 0.8265375857322264,
+      "learning_rate": 9.76891065322223e-06,
+      "loss": 1.7758,
+      "step": 3850
+    },
+    {
+      "epoch": 1.115720211700758,
+      "grad_norm": 0.8074509140095618,
+      "learning_rate": 9.762413546313597e-06,
+      "loss": 1.773,
+      "step": 3900
+    },
+    {
+      "epoch": 1.130024316978973,
+      "grad_norm": 1.2955529307297018,
+      "learning_rate": 9.755828842630269e-06,
+      "loss": 1.7653,
+      "step": 3950
+    },
+    {
+      "epoch": 1.144328422257188,
+      "grad_norm": 0.9723230317765355,
+      "learning_rate": 9.749156677494357e-06,
+      "loss": 1.7813,
+      "step": 4000
+    },
+    {
+      "epoch": 1.144328422257188,
+      "eval_loss": 1.7602086067199707,
+      "eval_runtime": 13.0715,
+      "eval_samples_per_second": 76.502,
+      "eval_steps_per_second": 2.448,
+      "step": 4000
+    },
+    {
+      "epoch": 1.1587755685881849,
+      "grad_norm": 1.6203576789230651,
+      "learning_rate": 9.742397188025394e-06,
+      "loss": 1.7709,
+      "step": 4050
+    },
+    {
+      "epoch": 1.1730796738663996,
+      "grad_norm": 1.1690135821413727,
+      "learning_rate": 9.735550513137513e-06,
+      "loss": 1.7583,
+      "step": 4100
+    },
+    {
+      "epoch": 1.1873837791446145,
+      "grad_norm": 1.2584964373503669,
+      "learning_rate": 9.728616793536588e-06,
+      "loss": 1.7653,
+      "step": 4150
+    },
+    {
+      "epoch": 1.2016878844228294,
+      "grad_norm": 1.022986798804273,
+      "learning_rate": 9.721596171717352e-06,
+      "loss": 1.7661,
+      "step": 4200
+    },
+    {
+      "epoch": 1.2159919897010443,
+      "grad_norm": 0.9715095488392225,
+      "learning_rate": 9.714488791960463e-06,
+      "loss": 1.7745,
+      "step": 4250
+    },
+    {
+      "epoch": 1.230296094979259,
+      "grad_norm": 1.1370122008173429,
+      "learning_rate": 9.707294800329536e-06,
+      "loss": 1.7684,
+      "step": 4300
+    },
+    {
+      "epoch": 1.2446002002574739,
+      "grad_norm": 0.9536116406695613,
+      "learning_rate": 9.700014344668152e-06,
+      "loss": 1.7606,
+      "step": 4350
+    },
+    {
+      "epoch": 1.2589043055356888,
+      "grad_norm": 0.7353885681465049,
+      "learning_rate": 9.692647574596803e-06,
+      "loss": 1.7633,
+      "step": 4400
+    },
+    {
+      "epoch": 1.2589043055356888,
+      "eval_loss": 1.752835988998413,
+      "eval_runtime": 13.2443,
+      "eval_samples_per_second": 75.504,
+      "eval_steps_per_second": 2.416,
+      "step": 4400
+    },
+    {
+      "epoch": 1.2732084108139037,
+      "grad_norm": 1.1802783991524877,
+      "learning_rate": 9.685194641509837e-06,
+      "loss": 1.7686,
+      "step": 4450
+    },
+    {
+      "epoch": 1.2875125160921184,
+      "grad_norm": 0.9318456023565893,
+      "learning_rate": 9.677655698572326e-06,
+      "loss": 1.7556,
+      "step": 4500
+    },
+    {
+      "epoch": 1.3018166213703333,
+      "grad_norm": 1.0437155245378358,
+      "learning_rate": 9.670030900716941e-06,
+      "loss": 1.7571,
+      "step": 4550
+    },
+    {
+      "epoch": 1.3161207266485482,
+      "grad_norm": 1.0912943460238356,
+      "learning_rate": 9.662320404640743e-06,
+      "loss": 1.7546,
+      "step": 4600
+    },
+    {
+      "epoch": 1.3304248319267629,
+      "grad_norm": 1.0063174963239234,
+      "learning_rate": 9.654524368801982e-06,
+      "loss": 1.7644,
+      "step": 4650
+    },
+    {
+      "epoch": 1.3447289372049778,
+      "grad_norm": 1.0417193249219145,
+      "learning_rate": 9.646642953416835e-06,
+      "loss": 1.7525,
+      "step": 4700
+    },
+    {
+      "epoch": 1.3590330424831927,
+      "grad_norm": 0.698422675064761,
+      "learning_rate": 9.638676320456109e-06,
+      "loss": 1.7586,
+      "step": 4750
+    },
+    {
+      "epoch": 1.3733371477614076,
+      "grad_norm": 1.1910335445073554,
+      "learning_rate": 9.630624633641918e-06,
+      "loss": 1.7528,
+      "step": 4800
+    },
+    {
+      "epoch": 1.3733371477614076,
+      "eval_loss": 1.7467565536499023,
+      "eval_runtime": 13.1696,
+      "eval_samples_per_second": 75.932,
+      "eval_steps_per_second": 2.43,
+      "step": 4800
+    },
+    {
+      "epoch": 1.3876412530396225,
+      "grad_norm": 0.860266713830524,
+      "learning_rate": 9.622488058444313e-06,
+      "loss": 1.7526,
+      "step": 4850
+    },
+    {
+      "epoch": 1.4019453583178372,
+      "grad_norm": 0.6381947527857678,
+      "learning_rate": 9.614266762077891e-06,
+      "loss": 1.7548,
+      "step": 4900
+    },
+    {
+      "epoch": 1.416249463596052,
+      "grad_norm": 0.7554075411255639,
+      "learning_rate": 9.605960913498342e-06,
+      "loss": 1.7481,
+      "step": 4950
+    },
+    {
+      "epoch": 1.430553568874267,
+      "grad_norm": 1.2154500150186984,
+      "learning_rate": 9.597570683398996e-06,
+      "loss": 1.7584,
+      "step": 5000
+    },
+    {
+      "epoch": 1.4448576741524817,
+      "grad_norm": 0.6439874478006935,
+      "learning_rate": 9.5890962442073e-06,
+      "loss": 1.7445,
+      "step": 5050
+    },
+    {
+      "epoch": 1.4591617794306966,
+      "grad_norm": 1.1984560510642397,
+      "learning_rate": 9.580537770081285e-06,
+      "loss": 1.7442,
+      "step": 5100
+    },
+    {
+      "epoch": 1.4734658847089115,
+      "grad_norm": 1.1871050477497047,
+      "learning_rate": 9.57189543690598e-06,
+      "loss": 1.7498,
+      "step": 5150
+    },
+    {
+      "epoch": 1.4877699899871262,
+      "grad_norm": 0.7133766609729304,
+      "learning_rate": 9.563169422289798e-06,
+      "loss": 1.7423,
+      "step": 5200
+    },
+    {
+      "epoch": 1.4877699899871262,
+      "eval_loss": 1.7412633895874023,
+      "eval_runtime": 13.1818,
+      "eval_samples_per_second": 75.862,
+      "eval_steps_per_second": 2.428,
+      "step": 5200
+    },
+    {
+      "epoch": 1.5020740952653413,
+      "grad_norm": 0.7612313099084432,
+      "learning_rate": 9.554359905560887e-06,
+      "loss": 1.748,
+      "step": 5250
+    },
+    {
+      "epoch": 1.516378200543556,
+      "grad_norm": 0.750224706069401,
+      "learning_rate": 9.54546706776345e-06,
+      "loss": 1.7435,
+      "step": 5300
+    },
+    {
+      "epoch": 1.530682305821771,
+      "grad_norm": 0.9331760005679197,
+      "learning_rate": 9.536491091654018e-06,
+      "loss": 1.7324,
+      "step": 5350
+    },
+    {
+      "epoch": 1.5449864110999858,
+      "grad_norm": 0.8613210624917503,
+      "learning_rate": 9.527432161697696e-06,
+      "loss": 1.751,
+      "step": 5400
+    },
+    {
+      "epoch": 1.5592905163782005,
+      "grad_norm": 4.862833366342641,
+      "learning_rate": 9.518290464064365e-06,
+      "loss": 1.7365,
+      "step": 5450
+    },
+    {
+      "epoch": 1.5735946216564154,
+      "grad_norm": 0.7838749485611254,
+      "learning_rate": 9.509066186624872e-06,
+      "loss": 1.7399,
+      "step": 5500
+    },
+    {
+      "epoch": 1.5878987269346303,
+      "grad_norm": 0.798441318649249,
+      "learning_rate": 9.499759518947156e-06,
+      "loss": 1.737,
+      "step": 5550
+    },
+    {
+      "epoch": 1.602202832212845,
+      "grad_norm": 0.7078515946606195,
+      "learning_rate": 9.490370652292357e-06,
+      "loss": 1.7412,
+      "step": 5600
+    },
+    {
+      "epoch": 1.602202832212845,
+      "eval_loss": 1.7350177764892578,
+      "eval_runtime": 13.1662,
+      "eval_samples_per_second": 75.952,
+      "eval_steps_per_second": 2.43,
+      "step": 5600
+    },
+    {
+      "epoch": 1.61650693749106,
+      "grad_norm": 0.8939699763780663,
+      "learning_rate": 9.480899779610883e-06,
+      "loss": 1.7485,
+      "step": 5650
+    },
+    {
+      "epoch": 1.6308110427692748,
+      "grad_norm": 0.7241948247907901,
+      "learning_rate": 9.471347095538448e-06,
+      "loss": 1.7338,
+      "step": 5700
+    },
+    {
+      "epoch": 1.6451151480474895,
+      "grad_norm": 0.7753555958595959,
+      "learning_rate": 9.461712796392067e-06,
+      "loss": 1.7423,
+      "step": 5750
+    },
+    {
+      "epoch": 1.6594192533257046,
+      "grad_norm": 0.9453545668822089,
+      "learning_rate": 9.45199708016603e-06,
+      "loss": 1.7411,
+      "step": 5800
+    },
+    {
+      "epoch": 1.6737233586039193,
+      "grad_norm": 0.8069354542585767,
+      "learning_rate": 9.442200146527824e-06,
+      "loss": 1.7341,
+      "step": 5850
+    },
+    {
+      "epoch": 1.6880274638821342,
+      "grad_norm": 0.9570095002200463,
+      "learning_rate": 9.432322196814032e-06,
+      "loss": 1.7309,
+      "step": 5900
+    },
+    {
+      "epoch": 1.7023315691603491,
+      "grad_norm": 0.778079342980594,
+      "learning_rate": 9.422363434026205e-06,
+      "loss": 1.7331,
+      "step": 5950
+    },
+    {
+      "epoch": 1.7166356744385638,
+      "grad_norm": 0.8582782820893037,
+      "learning_rate": 9.41232406282667e-06,
+      "loss": 1.7375,
+      "step": 6000
+    },
+    {
+      "epoch": 1.7166356744385638,
+      "eval_loss": 1.7310324907302856,
+      "eval_runtime": 13.1951,
+      "eval_samples_per_second": 75.786,
+      "eval_steps_per_second": 2.425,
+      "step": 6000
+    },
+    {
+      "epoch": 1.7309397797167787,
+      "grad_norm": 0.657112371502439,
+      "learning_rate": 9.402204289534344e-06,
+      "loss": 1.725,
+      "step": 6050
+    },
+    {
+      "epoch": 1.7452438849949936,
+      "grad_norm": 1.1123814501158173,
+      "learning_rate": 9.392004322120484e-06,
+      "loss": 1.7303,
+      "step": 6100
+    },
+    {
+      "epoch": 1.7595479902732083,
+      "grad_norm": 0.9499171364049573,
+      "learning_rate": 9.381724370204414e-06,
+      "loss": 1.7203,
+      "step": 6150
+    },
+    {
+      "epoch": 1.7738520955514234,
+      "grad_norm": 1.095712190889447,
+      "learning_rate": 9.371364645049216e-06,
+      "loss": 1.7291,
+      "step": 6200
+    },
+    {
+      "epoch": 1.788156200829638,
+      "grad_norm": 1.0199371888229702,
+      "learning_rate": 9.360925359557397e-06,
+      "loss": 1.7155,
+      "step": 6250
+    },
+    {
+      "epoch": 1.8024603061078528,
+      "grad_norm": 0.827904086276593,
+      "learning_rate": 9.3504067282665e-06,
+      "loss": 1.7404,
+      "step": 6300
+    },
+    {
+      "epoch": 1.816764411386068,
+      "grad_norm": 0.8252103968718575,
+      "learning_rate": 9.339808967344701e-06,
+      "loss": 1.7275,
+      "step": 6350
+    },
+    {
+      "epoch": 1.8310685166642826,
+      "grad_norm": 0.8608721928591242,
+      "learning_rate": 9.329132294586374e-06,
+      "loss": 1.7257,
+      "step": 6400
+    },
+    {
+      "epoch": 1.8310685166642826,
+      "eval_loss": 1.7276860475540161,
+      "eval_runtime": 13.1438,
+      "eval_samples_per_second": 76.082,
+      "eval_steps_per_second": 2.435,
+      "step": 6400
+    },
+    {
+      "epoch": 1.8453726219424975,
+      "grad_norm": 1.0655198477534613,
+      "learning_rate": 9.318376929407606e-06,
+      "loss": 1.7219,
+      "step": 6450
+    },
+    {
+      "epoch": 1.8596767272207124,
+      "grad_norm": 0.8202058864526146,
+      "learning_rate": 9.307543092841688e-06,
+      "loss": 1.7219,
+      "step": 6500
+    },
+    {
+      "epoch": 1.873980832498927,
+      "grad_norm": 0.9119156406563158,
+      "learning_rate": 9.296631007534576e-06,
+      "loss": 1.7232,
+      "step": 6550
+    },
+    {
+      "epoch": 1.888284937777142,
+      "grad_norm": 0.8561460550361113,
+      "learning_rate": 9.285640897740316e-06,
+      "loss": 1.718,
+      "step": 6600
+    },
+    {
+      "epoch": 1.902589043055357,
+      "grad_norm": 0.9549020817113293,
+      "learning_rate": 9.27457298931643e-06,
+      "loss": 1.7348,
+      "step": 6650
+    },
+    {
+      "epoch": 1.9168931483335716,
+      "grad_norm": 0.6729589207861246,
+      "learning_rate": 9.263427509719287e-06,
+      "loss": 1.7175,
+      "step": 6700
+    },
+    {
+      "epoch": 1.9311972536117867,
+      "grad_norm": 0.709786130314777,
+      "learning_rate": 9.252204687999401e-06,
+      "loss": 1.7287,
+      "step": 6750
+    },
+    {
+      "epoch": 1.9455013588900014,
+      "grad_norm": 0.9827727167496969,
+      "learning_rate": 9.240904754796767e-06,
+      "loss": 1.7241,
+      "step": 6800
+    },
+    {
+      "epoch": 1.9455013588900014,
+      "eval_loss": 1.7200103998184204,
+      "eval_runtime": 13.113,
+      "eval_samples_per_second": 76.26,
+      "eval_steps_per_second": 2.44,
+      "step": 6800
+    },
+    {
+      "epoch": 1.9598054641682163,
+      "grad_norm": 0.8218927694104888,
+      "learning_rate": 9.22952794233608e-06,
+      "loss": 1.7334,
+      "step": 6850
+    },
+    {
+      "epoch": 1.9741095694464312,
+      "grad_norm": 0.9202146449681148,
+      "learning_rate": 9.218074484421977e-06,
+      "loss": 1.7176,
+      "step": 6900
+    },
+    {
+      "epoch": 1.988413674724646,
+      "grad_norm": 0.9757466555134802,
+      "learning_rate": 9.206544616434249e-06,
+      "loss": 1.716,
+      "step": 6950
+    },
+    {
+      "epoch": 2.0027177800028606,
+      "grad_norm": 0.9665391581844135,
+      "learning_rate": 9.194938575322973e-06,
+      "loss": 1.7104,
+      "step": 7000
+    },
+    {
+      "epoch": 2.0170218852810757,
+      "grad_norm": 0.747601621726994,
+      "learning_rate": 9.183256599603672e-06,
+      "loss": 1.7045,
+      "step": 7050
+    },
+    {
+      "epoch": 2.0313259905592904,
+      "grad_norm": 0.590573479620199,
+      "learning_rate": 9.171498929352388e-06,
+      "loss": 1.7012,
+      "step": 7100
+    },
+    {
+      "epoch": 2.0456300958375055,
+      "grad_norm": 0.7932008874801502,
+      "learning_rate": 9.159665806200766e-06,
+      "loss": 1.7017,
+      "step": 7150
+    },
+    {
+      "epoch": 2.0599342011157202,
+      "grad_norm": 0.824718775314277,
+      "learning_rate": 9.147757473331082e-06,
+      "loss": 1.7019,
+      "step": 7200
+    },
+    {
+      "epoch": 2.0599342011157202,
+      "eval_loss": 1.7177115678787231,
+      "eval_runtime": 13.1014,
+      "eval_samples_per_second": 76.328,
+      "eval_steps_per_second": 2.442,
+      "step": 7200
+    },
+    {
+      "epoch": 2.074238306393935,
+      "grad_norm": 1.080101464877567,
+      "learning_rate": 9.135774175471244e-06,
+      "loss": 1.7056,
+      "step": 7250
+    },
+    {
+      "epoch": 2.08854241167215,
+      "grad_norm": 0.6161967559479131,
+      "learning_rate": 9.123716158889765e-06,
+      "loss": 1.6892,
+      "step": 7300
+    },
+    {
+      "epoch": 2.1028465169503647,
+      "grad_norm": 0.6259364978680784,
+      "learning_rate": 9.111583671390697e-06,
+      "loss": 1.6876,
+      "step": 7350
+    },
+    {
+      "epoch": 2.1171506222285794,
+      "grad_norm": 0.9054346826665011,
+      "learning_rate": 9.09937696230855e-06,
+      "loss": 1.7004,
+      "step": 7400
+    },
+    {
+      "epoch": 2.1314547275067945,
+      "grad_norm": 0.8115910709185014,
+      "learning_rate": 9.087096282503152e-06,
+      "loss": 1.6951,
+      "step": 7450
+    },
+    {
+      "epoch": 2.1457588327850092,
+      "grad_norm": 0.939188480700838,
+      "learning_rate": 9.074741884354507e-06,
+      "loss": 1.698,
+      "step": 7500
+    },
+    {
+      "epoch": 2.1600629380632244,
+      "grad_norm": 0.5461813129993677,
+      "learning_rate": 9.062314021757603e-06,
+      "loss": 1.6953,
+      "step": 7550
+    },
+    {
+      "epoch": 2.174367043341439,
+      "grad_norm": 0.8915398126133827,
+      "learning_rate": 9.049812950117191e-06,
+      "loss": 1.6838,
+      "step": 7600
+    },
+    {
+      "epoch": 2.174367043341439,
+      "eval_loss": 1.7138569355010986,
+      "eval_runtime": 13.1252,
+      "eval_samples_per_second": 76.189,
+      "eval_steps_per_second": 2.438,
+      "step": 7600
+    },
+    {
+      "epoch": 2.1886711486196537,
+      "grad_norm": 0.8119718442892704,
+      "learning_rate": 9.037238926342544e-06,
+      "loss": 1.7012,
+      "step": 7650
+    },
+    {
+      "epoch": 2.202975253897869,
+      "grad_norm": 0.6869354062120884,
+      "learning_rate": 9.02459220884217e-06,
+      "loss": 1.6918,
+      "step": 7700
+    },
+    {
+      "epoch": 2.2172793591760835,
+      "grad_norm": 0.7341604750585072,
+      "learning_rate": 9.011873057518503e-06,
+      "loss": 1.699,
+      "step": 7750
+    },
+    {
+      "epoch": 2.2315834644542982,
+      "grad_norm": 0.8261410417964352,
+      "learning_rate": 8.999081733762568e-06,
+      "loss": 1.6993,
+      "step": 7800
+    },
+    {
+      "epoch": 2.2458875697325134,
+      "grad_norm": 0.7611803854476097,
+      "learning_rate": 8.986218500448598e-06,
+      "loss": 1.7069,
+      "step": 7850
+    },
+    {
+      "epoch": 2.260191675010728,
+      "grad_norm": 0.771284372878959,
+      "learning_rate": 8.973283621928644e-06,
+      "loss": 1.7018,
+      "step": 7900
+    },
+    {
+      "epoch": 2.2744957802889427,
+      "grad_norm": 0.6069248728630589,
+      "learning_rate": 8.96027736402713e-06,
+      "loss": 1.6894,
+      "step": 7950
+    },
+    {
+      "epoch": 2.288799885567158,
+      "grad_norm": 0.6935253796751046,
+      "learning_rate": 8.947199994035402e-06,
+      "loss": 1.6857,
+      "step": 8000
+    },
+    {
+      "epoch": 2.288799885567158,
+      "eval_loss": 1.709199070930481,
+      "eval_runtime": 13.105,
+      "eval_samples_per_second": 76.307,
+      "eval_steps_per_second": 2.442,
+      "step": 8000
+    },
+    {
+      "epoch": 2.3031039908453725,
+      "grad_norm": 0.6395079401863663,
+      "learning_rate": 8.934051780706226e-06,
+      "loss": 1.6867,
+      "step": 8050
+    },
+    {
+      "epoch": 2.3174080961235877,
+      "grad_norm": 0.8808561738320105,
+      "learning_rate": 8.920832994248268e-06,
+      "loss": 1.6947,
+      "step": 8100
+    },
+    {
+      "epoch": 2.3317122014018024,
+      "grad_norm": 0.6888071930360784,
+      "learning_rate": 8.907543906320542e-06,
+      "loss": 1.7005,
+      "step": 8150
+    },
+    {
+      "epoch": 2.346016306680017,
+      "grad_norm": 0.6515296032830782,
+      "learning_rate": 8.894184790026823e-06,
+      "loss": 1.686,
+      "step": 8200
+    },
+    {
+      "epoch": 2.360320411958232,
+      "grad_norm": 0.7401595322241383,
+      "learning_rate": 8.880755919910048e-06,
+      "loss": 1.6865,
+      "step": 8250
+    },
+    {
+      "epoch": 2.374624517236447,
+      "grad_norm": 0.6000159117604718,
+      "learning_rate": 8.867257571946646e-06,
+      "loss": 1.6996,
+      "step": 8300
+    },
+    {
+      "epoch": 2.3889286225146615,
+      "grad_norm": 0.6577366719572108,
+      "learning_rate": 8.853690023540898e-06,
+      "loss": 1.6929,
+      "step": 8350
+    },
+    {
+      "epoch": 2.4032327277928767,
+      "grad_norm": 0.7392515614712054,
+      "learning_rate": 8.840053553519216e-06,
+      "loss": 1.6848,
+      "step": 8400
+    },
+    {
+      "epoch": 2.4032327277928767,
+      "eval_loss": 1.7050341367721558,
+      "eval_runtime": 13.1691,
+      "eval_samples_per_second": 75.935,
+      "eval_steps_per_second": 2.43,
+      "step": 8400
+    },
+    {
+      "epoch": 2.4175368330710914,
+      "grad_norm": 0.9889920645015057,
+      "learning_rate": 8.82634844212442e-06,
+      "loss": 1.6893,
+      "step": 8450
+    },
+    {
+      "epoch": 2.431840938349306,
+      "grad_norm": 0.9590160412926145,
+      "learning_rate": 8.81257497100998e-06,
+      "loss": 1.6846,
+      "step": 8500
+    },
+    {
+      "epoch": 2.446145043627521,
+      "grad_norm": 0.8138219488107618,
+      "learning_rate": 8.79873342323422e-06,
+      "loss": 1.689,
+      "step": 8550
+    },
+    {
+      "epoch": 2.460449148905736,
+      "grad_norm": 0.6771189511098283,
+      "learning_rate": 8.78482408325451e-06,
+      "loss": 1.684,
+      "step": 8600
+    },
+    {
+      "epoch": 2.474753254183951,
+      "grad_norm": 0.637569459443737,
+      "learning_rate": 8.770847236921412e-06,
+      "loss": 1.6893,
+      "step": 8650
+    },
+    {
+      "epoch": 2.4890573594621657,
+      "grad_norm": 0.8520093583138391,
+      "learning_rate": 8.756803171472817e-06,
+      "loss": 1.6858,
+      "step": 8700
+    },
+    {
+      "epoch": 2.5033614647403803,
+      "grad_norm": 1.0853751747133211,
+      "learning_rate": 8.742692175528027e-06,
+      "loss": 1.6929,
+      "step": 8750
+    },
+    {
+      "epoch": 2.5176655700185955,
+      "grad_norm": 0.5845518675223795,
+      "learning_rate": 8.728514539081837e-06,
+      "loss": 1.6795,
+      "step": 8800
+    },
+    {
+      "epoch": 2.5176655700185955,
+      "eval_loss": 1.6996084451675415,
+      "eval_runtime": 13.1313,
+      "eval_samples_per_second": 76.154,
+      "eval_steps_per_second": 2.437,
+      "step": 8800
+    },
+    {
+      "epoch": 2.53196967529681,
+      "grad_norm": 0.6987736172220224,
+      "learning_rate": 8.714270553498567e-06,
+      "loss": 1.6786,
+      "step": 8850
+    },
+    {
+      "epoch": 2.5462737805750253,
+      "grad_norm": 0.6845251902273746,
+      "learning_rate": 8.699960511506077e-06,
+      "loss": 1.6915,
+      "step": 8900
+    },
+    {
+      "epoch": 2.56057788585324,
+      "grad_norm": 0.5813491957603439,
+      "learning_rate": 8.685584707189749e-06,
+      "loss": 1.6864,
+      "step": 8950
+    },
+    {
+      "epoch": 2.5748819911314547,
+      "grad_norm": 0.7758297556506881,
+      "learning_rate": 8.671143435986447e-06,
+      "loss": 1.6853,
+      "step": 9000
+    },
+    {
+      "epoch": 2.5891860964096693,
+      "grad_norm": 0.6480784306311093,
+      "learning_rate": 8.656636994678447e-06,
+      "loss": 1.6945,
+      "step": 9050
+    },
+    {
+      "epoch": 2.6034902016878845,
+      "grad_norm": 0.752842227645809,
+      "learning_rate": 8.642065681387329e-06,
+      "loss": 1.6856,
+      "step": 9100
+    },
+    {
+      "epoch": 2.617794306966099,
+      "grad_norm": 0.8563670508754716,
+      "learning_rate": 8.627429795567858e-06,
+      "loss": 1.6813,
+      "step": 9150
+    },
+    {
+      "epoch": 2.6320984122443143,
+      "grad_norm": 0.649858543338678,
+      "learning_rate": 8.61272963800183e-06,
+      "loss": 1.6807,
+      "step": 9200
+    },
+    {
+      "epoch": 2.6320984122443143,
+      "eval_loss": 1.6953411102294922,
+      "eval_runtime": 13.1331,
+      "eval_samples_per_second": 76.143,
+      "eval_steps_per_second": 2.437,
+      "step": 9200
+    },
+    {
+      "epoch": 2.646402517522529,
+      "grad_norm": 0.6485355884487186,
+      "learning_rate": 8.597965510791883e-06,
+      "loss": 1.691,
+      "step": 9250
+    },
+    {
+      "epoch": 2.6607066228007437,
+      "grad_norm": 0.8331706908409144,
+      "learning_rate": 8.5831377173553e-06,
+      "loss": 1.6757,
+      "step": 9300
+    },
+    {
+      "epoch": 2.675010728078959,
+      "grad_norm": 0.7237339174259747,
+      "learning_rate": 8.568246562417762e-06,
+      "loss": 1.6838,
+      "step": 9350
+    },
+    {
+      "epoch": 2.6893148333571735,
+      "grad_norm": 0.6551466282285732,
+      "learning_rate": 8.553292352007096e-06,
+      "loss": 1.6815,
+      "step": 9400
+    },
+    {
+      "epoch": 2.7036189386353886,
+      "grad_norm": 0.6601355008192104,
+      "learning_rate": 8.538275393446976e-06,
+      "loss": 1.6857,
+      "step": 9450
+    },
+    {
+      "epoch": 2.7179230439136033,
+      "grad_norm": 0.6614316555641063,
+      "learning_rate": 8.523195995350613e-06,
+      "loss": 1.6823,
+      "step": 9500
+    },
+    {
+      "epoch": 2.732227149191818,
+      "grad_norm": 0.9043872683705563,
+      "learning_rate": 8.508054467614417e-06,
+      "loss": 1.6807,
+      "step": 9550
+    },
+    {
+      "epoch": 2.7465312544700327,
+      "grad_norm": 0.6024321505835114,
+      "learning_rate": 8.492851121411614e-06,
+      "loss": 1.6775,
+      "step": 9600
+    },
+    {
+      "epoch": 2.7465312544700327,
+      "eval_loss": 1.6916736364364624,
+      "eval_runtime": 13.1667,
+      "eval_samples_per_second": 75.949,
+      "eval_steps_per_second": 2.43,
+      "step": 9600
+    },
+    {
+      "epoch": 2.760835359748248,
+      "grad_norm": 0.5885102755974863,
+      "learning_rate": 8.477586269185868e-06,
+      "loss": 1.6783,
+      "step": 9650
+    },
+    {
+      "epoch": 2.7751394650264625,
+      "grad_norm": 0.5923048948697456,
+      "learning_rate": 8.462260224644848e-06,
+      "loss": 1.6754,
+      "step": 9700
+    },
+    {
+      "epoch": 2.7894435703046776,
+      "grad_norm": 0.658689052183787,
+      "learning_rate": 8.446873302753783e-06,
+      "loss": 1.6879,
+      "step": 9750
+    },
+    {
+      "epoch": 2.8037476755828923,
+      "grad_norm": 0.6839558742874721,
+      "learning_rate": 8.431425819728998e-06,
+      "loss": 1.6833,
+      "step": 9800
+    },
+    {
+      "epoch": 2.818051780861107,
+      "grad_norm": 0.563076108100303,
+      "learning_rate": 8.415918093031403e-06,
+      "loss": 1.6746,
+      "step": 9850
+    },
+    {
+      "epoch": 2.832355886139322,
+      "grad_norm": 0.6412513978425523,
+      "learning_rate": 8.400350441359976e-06,
+      "loss": 1.6673,
+      "step": 9900
+    },
+    {
+      "epoch": 2.846659991417537,
+      "grad_norm": 0.6509884557319101,
+      "learning_rate": 8.384723184645211e-06,
+      "loss": 1.6736,
+      "step": 9950
+    },
+    {
+      "epoch": 2.860964096695752,
+      "grad_norm": 0.6795285605676114,
+      "learning_rate": 8.369036644042546e-06,
+      "loss": 1.6602,
+      "step": 10000
+    },
+    {
+      "epoch": 2.860964096695752,
+      "eval_loss": 1.6880682706832886,
+      "eval_runtime": 13.1264,
+      "eval_samples_per_second": 76.182,
+      "eval_steps_per_second": 2.438,
+      "step": 10000
+    },
+    {
+      "epoch": 2.8752682019739666,
+      "grad_norm": 0.6331270907140948,
+      "learning_rate": 8.353291141925763e-06,
+      "loss": 1.6678,
+      "step": 10050
+    },
+    {
+      "epoch": 2.8895723072521813,
+      "grad_norm": 0.6165703754864105,
+      "learning_rate": 8.337487001880353e-06,
+      "loss": 1.6779,
+      "step": 10100
+    },
+    {
+      "epoch": 2.903876412530396,
+      "grad_norm": 0.9180760124515276,
+      "learning_rate": 8.32162454869688e-06,
+      "loss": 1.6709,
+      "step": 10150
+    },
+    {
+      "epoch": 2.918180517808611,
+      "grad_norm": 0.6067014553628142,
+      "learning_rate": 8.305704108364301e-06,
+      "loss": 1.6748,
+      "step": 10200
+    },
+    {
+      "epoch": 2.932484623086826,
+      "grad_norm": 0.6822167283880958,
+      "learning_rate": 8.289726008063265e-06,
+      "loss": 1.6734,
+      "step": 10250
+    },
+    {
+      "epoch": 2.946788728365041,
+      "grad_norm": 0.5414334397184379,
+      "learning_rate": 8.273690576159383e-06,
+      "loss": 1.674,
+      "step": 10300
+    },
+    {
+      "epoch": 2.9610928336432556,
+      "grad_norm": 0.6960756922615432,
+      "learning_rate": 8.257598142196496e-06,
+      "loss": 1.6611,
+      "step": 10350
+    },
+    {
+      "epoch": 2.9753969389214703,
+      "grad_norm": 0.6604617349648914,
+      "learning_rate": 8.241449036889892e-06,
+      "loss": 1.6676,
+      "step": 10400
+    },
+    {
+      "epoch": 2.9753969389214703,
+      "eval_loss": 1.6850733757019043,
+      "eval_runtime": 13.109,
+      "eval_samples_per_second": 76.284,
+      "eval_steps_per_second": 2.441,
+      "step": 10400
+    },
+    {
+      "epoch": 2.9897010441996854,
+      "grad_norm": 0.803147857087257,
+      "learning_rate": 8.225243592119501e-06,
+      "loss": 1.6742,
+      "step": 10450
+    },
+    {
+      "epoch": 3.0040051494779,
+      "grad_norm": 0.7661508794366779,
+      "learning_rate": 8.208982140923095e-06,
+      "loss": 1.6643,
+      "step": 10500
+    },
+    {
+      "epoch": 3.0183092547561152,
+      "grad_norm": 0.9234438361457962,
+      "learning_rate": 8.192665017489431e-06,
+      "loss": 1.6358,
+      "step": 10550
+    },
+    {
+      "epoch": 3.03261336003433,
+      "grad_norm": 0.6961614825939386,
+      "learning_rate": 8.17629255715138e-06,
+      "loss": 1.6545,
+      "step": 10600
+    },
+    {
+      "epoch": 3.0469174653125446,
+      "grad_norm": 0.6895369107938377,
+      "learning_rate": 8.159865096379046e-06,
+      "loss": 1.6321,
+      "step": 10650
+    },
+    {
+      "epoch": 3.0612215705907597,
+      "grad_norm": 0.7078335937537477,
+      "learning_rate": 8.14338297277284e-06,
+      "loss": 1.6349,
+      "step": 10700
+    },
+    {
+      "epoch": 3.0755256758689744,
+      "grad_norm": 0.6076483839068002,
+      "learning_rate": 8.126846525056555e-06,
+      "loss": 1.6365,
+      "step": 10750
+    },
+    {
+      "epoch": 3.089829781147189,
+      "grad_norm": 0.6898999480835771,
+      "learning_rate": 8.110256093070393e-06,
+      "loss": 1.6546,
+      "step": 10800
+    },
+    {
+      "epoch": 3.089829781147189,
+      "eval_loss": 1.6832627058029175,
+      "eval_runtime": 13.1347,
+      "eval_samples_per_second": 76.134,
+      "eval_steps_per_second": 2.436,
+      "step": 10800
+    },
+    {
+      "epoch": 3.104133886425404,
+      "grad_norm": 0.7013516335683451,
+      "learning_rate": 8.093612017763986e-06,
+      "loss": 1.639,
+      "step": 10850
+    },
+    {
+      "epoch": 3.118437991703619,
+      "grad_norm": 0.6585546257998172,
+      "learning_rate": 8.076914641189388e-06,
+      "loss": 1.649,
+      "step": 10900
+    },
+    {
+      "epoch": 3.1327420969818336,
+      "grad_norm": 0.6634919864983012,
+      "learning_rate": 8.060164306494052e-06,
+      "loss": 1.6349,
+      "step": 10950
+    },
+    {
+      "epoch": 3.1470462022600487,
+      "grad_norm": 0.6535125345700776,
+      "learning_rate": 8.043361357913763e-06,
+      "loss": 1.6354,
+      "step": 11000
+    },
+    {
+      "epoch": 3.1613503075382634,
+      "grad_norm": 0.6365757392209358,
+      "learning_rate": 8.026506140765581e-06,
+      "loss": 1.6411,
+      "step": 11050
+    },
+    {
+      "epoch": 3.1756544128164785,
+      "grad_norm": 0.7613496780923112,
+      "learning_rate": 8.009599001440733e-06,
+      "loss": 1.6443,
+      "step": 11100
+    },
+    {
+      "epoch": 3.189958518094693,
+      "grad_norm": 0.6788371997973981,
+      "learning_rate": 7.992640287397498e-06,
+      "loss": 1.6406,
+      "step": 11150
+    },
+    {
+      "epoch": 3.204262623372908,
+      "grad_norm": 0.7267161462233287,
+      "learning_rate": 7.975630347154062e-06,
+      "loss": 1.6416,
+      "step": 11200
+    },
+    {
+      "epoch": 3.204262623372908,
+      "eval_loss": 1.6800661087036133,
+      "eval_runtime": 13.1106,
+      "eval_samples_per_second": 76.274,
+      "eval_steps_per_second": 2.441,
+      "step": 11200
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 34950,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 800,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9959545317097472.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83637d8dedf8ccf60a105628fc1cf9223c41f25068decae4ba1fdc6718654cc4
+size 7224