fc_embedding.weight q8_0 2560 2560 0 6963200 fc_hidden.weight q8_0 2560 2560 6963200 6963200 hyper_connection_mixer.hc_norm.weight f32 1 10240 13926400 40960 hyper_connection_mixer.input_mix_weight_down.weight bf16 320 10240 13967360 6553600 hyper_connection_mixer.input_mix_weight_up.weight bf16 10240 320 20520960 6553600 attn_hyper_connection.block_inject_weight.weight bf16 4 10240 27074560 81920 attn_hyper_connection.hc_norm.weight f32 1 10240 27156480 40960 attn_hyper_connection.input_mix_weight_down.weight bf16 320 10240 27197440 6553600 attn_hyper_connection.input_mix_weight_up.weight bf16 10240 320 33751040 6553600 mlp.gate.weight bf16 512 2560 40304640 2621440 mlp.shared_expert.down_proj.weight q8_0 2560 640 42926080 1740800 mlp.shared_expert.gate_proj.weight q8_0 640 2560 44666880 1740800 mlp.shared_expert.up_proj.weight q8_0 640 2560 46407680 1740800 mlp.shared_expert_gate.weight bf16 1 2560 48148480 5120 mlp_hyper_connection.block_inject_weight.weight bf16 4 10240 48153600 81920 mlp_hyper_connection.hc_norm.weight f32 1 10240 48235520 40960 mlp_hyper_connection.input_mix_weight_down.weight bf16 320 10240 48276480 6553600 mlp_hyper_connection.input_mix_weight_up.weight bf16 10240 320 54830080 6553600 self_attn.indexer.index_qk_proj.weight q8_0 640 2560 61383680 1740800 self_attn.indexer.k_layernorm.weight f32 1 128 63124480 512 self_attn.indexer.q_layernorm.weight f32 1 128 63124992 512 self_attn.k_norm.weight f32 1 256 63125504 1024 self_attn.k_proj.weight q8_0 512 2560 63126528 1392640 self_attn.o_proj.weight q8_0 2560 6144 64519168 16711680 self_attn.q_norm.weight f32 1 256 81230848 1024 self_attn.q_proj.weight q8_0 12288 2560 81231872 33423360 self_attn.v_proj.weight q8_0 512 2560 114655232 1392640 pre_fc_norm_embedding.weight f32 1 2560 116047872 10240 pre_fc_norm_hidden.weight f32 1 10240 116058112 40960