{ "format": "zonos3-web-v1", "model": "teacher", "meta": { "general.architecture": "w2vbert_ssl", "general.name": "Matched Wav2Vec2-BERT semantic teacher layer 16", "sample_rate": 16000, "max_seconds": 60, "n_layer": 16, "n_embd": 1024, "n_head": 16, "n_ff": 4096, "rel_left": 64, "rel_right": 8, "conv_kernel": 31, "n_mels": 80, "frame_length": 400, "hop_length": 160, "fft_length": 512, "stride": 2, "eps": 9.999999747378752e-06, "preemphasis": 0.9700000286102295, "mel_floor": 1.1920928955078125e-07, "padding_value": 1.0 }, "shards": [ "shard-000.bin", "shard-001.bin", "shard-002.bin", "shard-003.bin", "shard-004.bin", "shard-005.bin", "shard-006.bin", "shard-007.bin", "shard-008.bin", "shard-009.bin", "shard-010.bin", "shard-011.bin" ], "tensors": { "frontend.window": { "dtype": "f32", "shape": [ 400 ], "shard": 0, "offset": 0, "nbytes": 1600, "host": true }, "frontend.mel_filters": { "dtype": "f32", "shape": [ 80, 257 ], "shard": 0, "offset": 1792, "nbytes": 82240, "host": true }, "frontend.resample_44100": { "dtype": "f32", "shape": [ 475, 160 ], "shard": 0, "offset": 84224, "nbytes": 304000, "host": true }, "fp.ln.weight": { "dtype": "f32", "shape": [ 160 ], "shard": 0, "offset": 388352, "nbytes": 640 }, "fp.ln.bias": { "dtype": "f32", "shape": [ 160 ], "shard": 0, "offset": 389120, "nbytes": 640 }, "fp.proj.weight": { "dtype": "bf16", "shape": [ 160, 1024 ], "shard": 0, "offset": 389888, "nbytes": 327680 }, "fp.proj.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 717568, "nbytes": 4096 }, "enc.0.ffn1_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 721664, "nbytes": 4096 }, "enc.0.ffn1_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 725760, "nbytes": 4096 }, "enc.0.ffn1.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 0, "offset": 729856, "nbytes": 8388608 }, "enc.0.ffn1.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 0, "offset": 9118464, "nbytes": 16384 }, "enc.0.ffn1.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 0, "offset": 9134848, "nbytes": 8388608 }, "enc.0.ffn1.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 17523456, "nbytes": 4096 }, "enc.0.ffn2_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 17527552, "nbytes": 4096 }, "enc.0.ffn2_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 17531648, "nbytes": 4096 }, "enc.0.ffn2.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 0, "offset": 17535744, "nbytes": 8388608 }, "enc.0.ffn2.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 0, "offset": 25924352, "nbytes": 16384 }, "enc.0.ffn2.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 0, "offset": 25940736, "nbytes": 8388608 }, "enc.0.ffn2.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 34329344, "nbytes": 4096 }, "enc.0.attn_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 34333440, "nbytes": 4096 }, "enc.0.attn_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 34337536, "nbytes": 4096 }, "enc.0.attn.q.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 0, "offset": 34341632, "nbytes": 2097152 }, "enc.0.attn.q.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 36438784, "nbytes": 4096 }, "enc.0.attn.k.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 0, "offset": 36442880, "nbytes": 2097152 }, "enc.0.attn.k.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 38540032, "nbytes": 4096 }, "enc.0.attn.v.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 0, "offset": 38544128, "nbytes": 2097152 }, "enc.0.attn.v.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 40641280, "nbytes": 4096 }, "enc.0.attn.o.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 0, "offset": 40645376, "nbytes": 2097152 }, "enc.0.attn.o.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 42742528, "nbytes": 4096 }, "enc.0.attn.dist": { "dtype": "f32", "shape": [ 64, 73 ], "shard": 0, "offset": 42746624, "nbytes": 18688 }, "enc.0.conv_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 42765312, "nbytes": 4096 }, "enc.0.conv_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 42769408, "nbytes": 4096 }, "enc.0.conv.pw1.weight": { "dtype": "bf16", "shape": [ 1024, 2048 ], "shard": 0, "offset": 42773504, "nbytes": 4194304 }, "enc.0.conv.dw.weight": { "dtype": "f32", "shape": [ 1024, 31 ], "shard": 0, "offset": 46967808, "nbytes": 126976 }, "enc.0.conv.dw_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 47094784, "nbytes": 4096 }, "enc.0.conv.dw_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 47098880, "nbytes": 4096 }, "enc.0.conv.pw2.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 0, "offset": 47102976, "nbytes": 2097152 }, "enc.0.final_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 49200128, "nbytes": 4096 }, "enc.0.final_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 49204224, "nbytes": 4096 }, "enc.1.ffn1_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 49208320, "nbytes": 4096 }, "enc.1.ffn1_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 49212416, "nbytes": 4096 }, "enc.1.ffn1.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 0, "offset": 49216512, "nbytes": 8388608 }, "enc.1.ffn1.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 0, "offset": 57605120, "nbytes": 16384 }, "enc.1.ffn1.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 0, "offset": 57621504, "nbytes": 8388608 }, "enc.1.ffn1.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 66010112, "nbytes": 4096 }, "enc.1.ffn2_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 66014208, "nbytes": 4096 }, "enc.1.ffn2_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 0, "offset": 66018304, "nbytes": 4096 }, "enc.1.ffn2.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 1, "offset": 0, "nbytes": 8388608 }, "enc.1.ffn2.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 1, "offset": 8388608, "nbytes": 16384 }, "enc.1.ffn2.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 1, "offset": 8404992, "nbytes": 8388608 }, "enc.1.ffn2.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 16793600, "nbytes": 4096 }, "enc.1.attn_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 16797696, "nbytes": 4096 }, "enc.1.attn_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 16801792, "nbytes": 4096 }, "enc.1.attn.q.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 1, "offset": 16805888, "nbytes": 2097152 }, "enc.1.attn.q.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 18903040, "nbytes": 4096 }, "enc.1.attn.k.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 1, "offset": 18907136, "nbytes": 2097152 }, "enc.1.attn.k.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 21004288, "nbytes": 4096 }, "enc.1.attn.v.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 1, "offset": 21008384, "nbytes": 2097152 }, "enc.1.attn.v.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 23105536, "nbytes": 4096 }, "enc.1.attn.o.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 1, "offset": 23109632, "nbytes": 2097152 }, "enc.1.attn.o.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 25206784, "nbytes": 4096 }, "enc.1.attn.dist": { "dtype": "f32", "shape": [ 64, 73 ], "shard": 1, "offset": 25210880, "nbytes": 18688 }, "enc.1.conv_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 25229568, "nbytes": 4096 }, "enc.1.conv_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 25233664, "nbytes": 4096 }, "enc.1.conv.pw1.weight": { "dtype": "bf16", "shape": [ 1024, 2048 ], "shard": 1, "offset": 25237760, "nbytes": 4194304 }, "enc.1.conv.dw.weight": { "dtype": "f32", "shape": [ 1024, 31 ], "shard": 1, "offset": 29432064, "nbytes": 126976 }, "enc.1.conv.dw_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 29559040, "nbytes": 4096 }, "enc.1.conv.dw_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 29563136, "nbytes": 4096 }, "enc.1.conv.pw2.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 1, "offset": 29567232, "nbytes": 2097152 }, "enc.1.final_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 31664384, "nbytes": 4096 }, "enc.1.final_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 31668480, "nbytes": 4096 }, "enc.2.ffn1_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 31672576, "nbytes": 4096 }, "enc.2.ffn1_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 31676672, "nbytes": 4096 }, "enc.2.ffn1.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 1, "offset": 31680768, "nbytes": 8388608 }, "enc.2.ffn1.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 1, "offset": 40069376, "nbytes": 16384 }, "enc.2.ffn1.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 1, "offset": 40085760, "nbytes": 8388608 }, "enc.2.ffn1.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 48474368, "nbytes": 4096 }, "enc.2.ffn2_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 48478464, "nbytes": 4096 }, "enc.2.ffn2_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 48482560, "nbytes": 4096 }, "enc.2.ffn2.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 1, "offset": 48486656, "nbytes": 8388608 }, "enc.2.ffn2.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 1, "offset": 56875264, "nbytes": 16384 }, "enc.2.ffn2.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 1, "offset": 56891648, "nbytes": 8388608 }, "enc.2.ffn2.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 65280256, "nbytes": 4096 }, "enc.2.attn_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 65284352, "nbytes": 4096 }, "enc.2.attn_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 1, "offset": 65288448, "nbytes": 4096 }, "enc.2.attn.q.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 2, "offset": 0, "nbytes": 2097152 }, "enc.2.attn.q.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 2097152, "nbytes": 4096 }, "enc.2.attn.k.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 2, "offset": 2101248, "nbytes": 2097152 }, "enc.2.attn.k.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 4198400, "nbytes": 4096 }, "enc.2.attn.v.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 2, "offset": 4202496, "nbytes": 2097152 }, "enc.2.attn.v.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 6299648, "nbytes": 4096 }, "enc.2.attn.o.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 2, "offset": 6303744, "nbytes": 2097152 }, "enc.2.attn.o.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 8400896, "nbytes": 4096 }, "enc.2.attn.dist": { "dtype": "f32", "shape": [ 64, 73 ], "shard": 2, "offset": 8404992, "nbytes": 18688 }, "enc.2.conv_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 8423680, "nbytes": 4096 }, "enc.2.conv_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 8427776, "nbytes": 4096 }, "enc.2.conv.pw1.weight": { "dtype": "bf16", "shape": [ 1024, 2048 ], "shard": 2, "offset": 8431872, "nbytes": 4194304 }, "enc.2.conv.dw.weight": { "dtype": "f32", "shape": [ 1024, 31 ], "shard": 2, "offset": 12626176, "nbytes": 126976 }, "enc.2.conv.dw_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 12753152, "nbytes": 4096 }, "enc.2.conv.dw_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 12757248, "nbytes": 4096 }, "enc.2.conv.pw2.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 2, "offset": 12761344, "nbytes": 2097152 }, "enc.2.final_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 14858496, "nbytes": 4096 }, "enc.2.final_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 14862592, "nbytes": 4096 }, "enc.3.ffn1_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 14866688, "nbytes": 4096 }, "enc.3.ffn1_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 14870784, "nbytes": 4096 }, "enc.3.ffn1.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 2, "offset": 14874880, "nbytes": 8388608 }, "enc.3.ffn1.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 2, "offset": 23263488, "nbytes": 16384 }, "enc.3.ffn1.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 2, "offset": 23279872, "nbytes": 8388608 }, "enc.3.ffn1.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 31668480, "nbytes": 4096 }, "enc.3.ffn2_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 31672576, "nbytes": 4096 }, "enc.3.ffn2_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 31676672, "nbytes": 4096 }, "enc.3.ffn2.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 2, "offset": 31680768, "nbytes": 8388608 }, "enc.3.ffn2.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 2, "offset": 40069376, "nbytes": 16384 }, "enc.3.ffn2.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 2, "offset": 40085760, "nbytes": 8388608 }, "enc.3.ffn2.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 48474368, "nbytes": 4096 }, "enc.3.attn_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 48478464, "nbytes": 4096 }, "enc.3.attn_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 48482560, "nbytes": 4096 }, "enc.3.attn.q.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 2, "offset": 48486656, "nbytes": 2097152 }, "enc.3.attn.q.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 50583808, "nbytes": 4096 }, "enc.3.attn.k.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 2, "offset": 50587904, "nbytes": 2097152 }, "enc.3.attn.k.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 52685056, "nbytes": 4096 }, "enc.3.attn.v.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 2, "offset": 52689152, "nbytes": 2097152 }, "enc.3.attn.v.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 54786304, "nbytes": 4096 }, "enc.3.attn.o.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 2, "offset": 54790400, "nbytes": 2097152 }, "enc.3.attn.o.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 56887552, "nbytes": 4096 }, "enc.3.attn.dist": { "dtype": "f32", "shape": [ 64, 73 ], "shard": 2, "offset": 56891648, "nbytes": 18688 }, "enc.3.conv_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 56910336, "nbytes": 4096 }, "enc.3.conv_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 56914432, "nbytes": 4096 }, "enc.3.conv.pw1.weight": { "dtype": "bf16", "shape": [ 1024, 2048 ], "shard": 2, "offset": 56918528, "nbytes": 4194304 }, "enc.3.conv.dw.weight": { "dtype": "f32", "shape": [ 1024, 31 ], "shard": 2, "offset": 61112832, "nbytes": 126976 }, "enc.3.conv.dw_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 61239808, "nbytes": 4096 }, "enc.3.conv.dw_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 61243904, "nbytes": 4096 }, "enc.3.conv.pw2.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 2, "offset": 61248000, "nbytes": 2097152 }, "enc.3.final_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 63345152, "nbytes": 4096 }, "enc.3.final_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 63349248, "nbytes": 4096 }, "enc.4.ffn1_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 63353344, "nbytes": 4096 }, "enc.4.ffn1_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 63357440, "nbytes": 4096 }, "enc.4.ffn1.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 3, "offset": 0, "nbytes": 8388608 }, "enc.4.ffn1.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 3, "offset": 8388608, "nbytes": 16384 }, "enc.4.ffn1.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 3, "offset": 8404992, "nbytes": 8388608 }, "enc.4.ffn1.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 16793600, "nbytes": 4096 }, "enc.4.ffn2_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 16797696, "nbytes": 4096 }, "enc.4.ffn2_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 16801792, "nbytes": 4096 }, "enc.4.ffn2.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 3, "offset": 16805888, "nbytes": 8388608 }, "enc.4.ffn2.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 3, "offset": 25194496, "nbytes": 16384 }, "enc.4.ffn2.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 3, "offset": 25210880, "nbytes": 8388608 }, "enc.4.ffn2.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 33599488, "nbytes": 4096 }, "enc.4.attn_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 33603584, "nbytes": 4096 }, "enc.4.attn_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 33607680, "nbytes": 4096 }, "enc.4.attn.q.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 3, "offset": 33611776, "nbytes": 2097152 }, "enc.4.attn.q.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 35708928, "nbytes": 4096 }, "enc.4.attn.k.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 3, "offset": 35713024, "nbytes": 2097152 }, "enc.4.attn.k.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 37810176, "nbytes": 4096 }, "enc.4.attn.v.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 3, "offset": 37814272, "nbytes": 2097152 }, "enc.4.attn.v.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 39911424, "nbytes": 4096 }, "enc.4.attn.o.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 3, "offset": 39915520, "nbytes": 2097152 }, "enc.4.attn.o.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 42012672, "nbytes": 4096 }, "enc.4.attn.dist": { "dtype": "f32", "shape": [ 64, 73 ], "shard": 3, "offset": 42016768, "nbytes": 18688 }, "enc.4.conv_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 42035456, "nbytes": 4096 }, "enc.4.conv_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 42039552, "nbytes": 4096 }, "enc.4.conv.pw1.weight": { "dtype": "bf16", "shape": [ 1024, 2048 ], "shard": 3, "offset": 42043648, "nbytes": 4194304 }, "enc.4.conv.dw.weight": { "dtype": "f32", "shape": [ 1024, 31 ], "shard": 3, "offset": 46237952, "nbytes": 126976 }, "enc.4.conv.dw_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 46364928, "nbytes": 4096 }, "enc.4.conv.dw_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 46369024, "nbytes": 4096 }, "enc.4.conv.pw2.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 3, "offset": 46373120, "nbytes": 2097152 }, "enc.4.final_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 48470272, "nbytes": 4096 }, "enc.4.final_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 48474368, "nbytes": 4096 }, "enc.5.ffn1_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 48478464, "nbytes": 4096 }, "enc.5.ffn1_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 48482560, "nbytes": 4096 }, "enc.5.ffn1.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 3, "offset": 48486656, "nbytes": 8388608 }, "enc.5.ffn1.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 3, "offset": 56875264, "nbytes": 16384 }, "enc.5.ffn1.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 3, "offset": 56891648, "nbytes": 8388608 }, "enc.5.ffn1.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 65280256, "nbytes": 4096 }, "enc.5.ffn2_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 65284352, "nbytes": 4096 }, "enc.5.ffn2_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 3, "offset": 65288448, "nbytes": 4096 }, "enc.5.ffn2.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 4, "offset": 0, "nbytes": 8388608 }, "enc.5.ffn2.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 4, "offset": 8388608, "nbytes": 16384 }, "enc.5.ffn2.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 4, "offset": 8404992, "nbytes": 8388608 }, "enc.5.ffn2.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 16793600, "nbytes": 4096 }, "enc.5.attn_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 16797696, "nbytes": 4096 }, "enc.5.attn_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 16801792, "nbytes": 4096 }, "enc.5.attn.q.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 4, "offset": 16805888, "nbytes": 2097152 }, "enc.5.attn.q.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 18903040, "nbytes": 4096 }, "enc.5.attn.k.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 4, "offset": 18907136, "nbytes": 2097152 }, "enc.5.attn.k.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 21004288, "nbytes": 4096 }, "enc.5.attn.v.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 4, "offset": 21008384, "nbytes": 2097152 }, "enc.5.attn.v.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 23105536, "nbytes": 4096 }, "enc.5.attn.o.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 4, "offset": 23109632, "nbytes": 2097152 }, "enc.5.attn.o.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 25206784, "nbytes": 4096 }, "enc.5.attn.dist": { "dtype": "f32", "shape": [ 64, 73 ], "shard": 4, "offset": 25210880, "nbytes": 18688 }, "enc.5.conv_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 25229568, "nbytes": 4096 }, "enc.5.conv_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 25233664, "nbytes": 4096 }, "enc.5.conv.pw1.weight": { "dtype": "bf16", "shape": [ 1024, 2048 ], "shard": 4, "offset": 25237760, "nbytes": 4194304 }, "enc.5.conv.dw.weight": { "dtype": "f32", "shape": [ 1024, 31 ], "shard": 4, "offset": 29432064, "nbytes": 126976 }, "enc.5.conv.dw_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 29559040, "nbytes": 4096 }, "enc.5.conv.dw_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 29563136, "nbytes": 4096 }, "enc.5.conv.pw2.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 4, "offset": 29567232, "nbytes": 2097152 }, "enc.5.final_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 31664384, "nbytes": 4096 }, "enc.5.final_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 31668480, "nbytes": 4096 }, "enc.6.ffn1_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 31672576, "nbytes": 4096 }, "enc.6.ffn1_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 31676672, "nbytes": 4096 }, "enc.6.ffn1.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 4, "offset": 31680768, "nbytes": 8388608 }, "enc.6.ffn1.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 4, "offset": 40069376, "nbytes": 16384 }, "enc.6.ffn1.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 4, "offset": 40085760, "nbytes": 8388608 }, "enc.6.ffn1.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 48474368, "nbytes": 4096 }, "enc.6.ffn2_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 48478464, "nbytes": 4096 }, "enc.6.ffn2_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 48482560, "nbytes": 4096 }, "enc.6.ffn2.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 4, "offset": 48486656, "nbytes": 8388608 }, "enc.6.ffn2.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 4, "offset": 56875264, "nbytes": 16384 }, "enc.6.ffn2.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 4, "offset": 56891648, "nbytes": 8388608 }, "enc.6.ffn2.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 65280256, "nbytes": 4096 }, "enc.6.attn_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 65284352, "nbytes": 4096 }, "enc.6.attn_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 4, "offset": 65288448, "nbytes": 4096 }, "enc.6.attn.q.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 5, "offset": 0, "nbytes": 2097152 }, "enc.6.attn.q.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 2097152, "nbytes": 4096 }, "enc.6.attn.k.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 5, "offset": 2101248, "nbytes": 2097152 }, "enc.6.attn.k.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 4198400, "nbytes": 4096 }, "enc.6.attn.v.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 5, "offset": 4202496, "nbytes": 2097152 }, "enc.6.attn.v.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 6299648, "nbytes": 4096 }, "enc.6.attn.o.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 5, "offset": 6303744, "nbytes": 2097152 }, "enc.6.attn.o.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 8400896, "nbytes": 4096 }, "enc.6.attn.dist": { "dtype": "f32", "shape": [ 64, 73 ], "shard": 5, "offset": 8404992, "nbytes": 18688 }, "enc.6.conv_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 8423680, "nbytes": 4096 }, "enc.6.conv_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 8427776, "nbytes": 4096 }, "enc.6.conv.pw1.weight": { "dtype": "bf16", "shape": [ 1024, 2048 ], "shard": 5, "offset": 8431872, "nbytes": 4194304 }, "enc.6.conv.dw.weight": { "dtype": "f32", "shape": [ 1024, 31 ], "shard": 5, "offset": 12626176, "nbytes": 126976 }, "enc.6.conv.dw_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 12753152, "nbytes": 4096 }, "enc.6.conv.dw_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 12757248, "nbytes": 4096 }, "enc.6.conv.pw2.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 5, "offset": 12761344, "nbytes": 2097152 }, "enc.6.final_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 14858496, "nbytes": 4096 }, "enc.6.final_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 14862592, "nbytes": 4096 }, "enc.7.ffn1_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 14866688, "nbytes": 4096 }, "enc.7.ffn1_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 14870784, "nbytes": 4096 }, "enc.7.ffn1.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 5, "offset": 14874880, "nbytes": 8388608 }, "enc.7.ffn1.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 5, "offset": 23263488, "nbytes": 16384 }, "enc.7.ffn1.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 5, "offset": 23279872, "nbytes": 8388608 }, "enc.7.ffn1.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 31668480, "nbytes": 4096 }, "enc.7.ffn2_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 31672576, "nbytes": 4096 }, "enc.7.ffn2_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 31676672, "nbytes": 4096 }, "enc.7.ffn2.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 5, "offset": 31680768, "nbytes": 8388608 }, "enc.7.ffn2.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 5, "offset": 40069376, "nbytes": 16384 }, "enc.7.ffn2.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 5, "offset": 40085760, "nbytes": 8388608 }, "enc.7.ffn2.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 48474368, "nbytes": 4096 }, "enc.7.attn_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 48478464, "nbytes": 4096 }, "enc.7.attn_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 48482560, "nbytes": 4096 }, "enc.7.attn.q.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 5, "offset": 48486656, "nbytes": 2097152 }, "enc.7.attn.q.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 50583808, "nbytes": 4096 }, "enc.7.attn.k.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 5, "offset": 50587904, "nbytes": 2097152 }, "enc.7.attn.k.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 52685056, "nbytes": 4096 }, "enc.7.attn.v.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 5, "offset": 52689152, "nbytes": 2097152 }, "enc.7.attn.v.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 54786304, "nbytes": 4096 }, "enc.7.attn.o.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 5, "offset": 54790400, "nbytes": 2097152 }, "enc.7.attn.o.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 56887552, "nbytes": 4096 }, "enc.7.attn.dist": { "dtype": "f32", "shape": [ 64, 73 ], "shard": 5, "offset": 56891648, "nbytes": 18688 }, "enc.7.conv_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 56910336, "nbytes": 4096 }, "enc.7.conv_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 56914432, "nbytes": 4096 }, "enc.7.conv.pw1.weight": { "dtype": "bf16", "shape": [ 1024, 2048 ], "shard": 5, "offset": 56918528, "nbytes": 4194304 }, "enc.7.conv.dw.weight": { "dtype": "f32", "shape": [ 1024, 31 ], "shard": 5, "offset": 61112832, "nbytes": 126976 }, "enc.7.conv.dw_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 61239808, "nbytes": 4096 }, "enc.7.conv.dw_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 61243904, "nbytes": 4096 }, "enc.7.conv.pw2.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 5, "offset": 61248000, "nbytes": 2097152 }, "enc.7.final_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 63345152, "nbytes": 4096 }, "enc.7.final_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 63349248, "nbytes": 4096 }, "enc.8.ffn1_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 63353344, "nbytes": 4096 }, "enc.8.ffn1_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 5, "offset": 63357440, "nbytes": 4096 }, "enc.8.ffn1.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 6, "offset": 0, "nbytes": 8388608 }, "enc.8.ffn1.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 6, "offset": 8388608, "nbytes": 16384 }, "enc.8.ffn1.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 6, "offset": 8404992, "nbytes": 8388608 }, "enc.8.ffn1.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 16793600, "nbytes": 4096 }, "enc.8.ffn2_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 16797696, "nbytes": 4096 }, "enc.8.ffn2_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 16801792, "nbytes": 4096 }, "enc.8.ffn2.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 6, "offset": 16805888, "nbytes": 8388608 }, "enc.8.ffn2.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 6, "offset": 25194496, "nbytes": 16384 }, "enc.8.ffn2.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 6, "offset": 25210880, "nbytes": 8388608 }, "enc.8.ffn2.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 33599488, "nbytes": 4096 }, "enc.8.attn_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 33603584, "nbytes": 4096 }, "enc.8.attn_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 33607680, "nbytes": 4096 }, "enc.8.attn.q.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 6, "offset": 33611776, "nbytes": 2097152 }, "enc.8.attn.q.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 35708928, "nbytes": 4096 }, "enc.8.attn.k.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 6, "offset": 35713024, "nbytes": 2097152 }, "enc.8.attn.k.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 37810176, "nbytes": 4096 }, "enc.8.attn.v.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 6, "offset": 37814272, "nbytes": 2097152 }, "enc.8.attn.v.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 39911424, "nbytes": 4096 }, "enc.8.attn.o.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 6, "offset": 39915520, "nbytes": 2097152 }, "enc.8.attn.o.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 42012672, "nbytes": 4096 }, "enc.8.attn.dist": { "dtype": "f32", "shape": [ 64, 73 ], "shard": 6, "offset": 42016768, "nbytes": 18688 }, "enc.8.conv_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 42035456, "nbytes": 4096 }, "enc.8.conv_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 42039552, "nbytes": 4096 }, "enc.8.conv.pw1.weight": { "dtype": "bf16", "shape": [ 1024, 2048 ], "shard": 6, "offset": 42043648, "nbytes": 4194304 }, "enc.8.conv.dw.weight": { "dtype": "f32", "shape": [ 1024, 31 ], "shard": 6, "offset": 46237952, "nbytes": 126976 }, "enc.8.conv.dw_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 46364928, "nbytes": 4096 }, "enc.8.conv.dw_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 46369024, "nbytes": 4096 }, "enc.8.conv.pw2.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 6, "offset": 46373120, "nbytes": 2097152 }, "enc.8.final_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 48470272, "nbytes": 4096 }, "enc.8.final_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 48474368, "nbytes": 4096 }, "enc.9.ffn1_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 48478464, "nbytes": 4096 }, "enc.9.ffn1_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 48482560, "nbytes": 4096 }, "enc.9.ffn1.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 6, "offset": 48486656, "nbytes": 8388608 }, "enc.9.ffn1.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 6, "offset": 56875264, "nbytes": 16384 }, "enc.9.ffn1.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 6, "offset": 56891648, "nbytes": 8388608 }, "enc.9.ffn1.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 65280256, "nbytes": 4096 }, "enc.9.ffn2_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 65284352, "nbytes": 4096 }, "enc.9.ffn2_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 6, "offset": 65288448, "nbytes": 4096 }, "enc.9.ffn2.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 7, "offset": 0, "nbytes": 8388608 }, "enc.9.ffn2.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 7, "offset": 8388608, "nbytes": 16384 }, "enc.9.ffn2.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 7, "offset": 8404992, "nbytes": 8388608 }, "enc.9.ffn2.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 16793600, "nbytes": 4096 }, "enc.9.attn_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 16797696, "nbytes": 4096 }, "enc.9.attn_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 16801792, "nbytes": 4096 }, "enc.9.attn.q.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 7, "offset": 16805888, "nbytes": 2097152 }, "enc.9.attn.q.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 18903040, "nbytes": 4096 }, "enc.9.attn.k.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 7, "offset": 18907136, "nbytes": 2097152 }, "enc.9.attn.k.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 21004288, "nbytes": 4096 }, "enc.9.attn.v.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 7, "offset": 21008384, "nbytes": 2097152 }, "enc.9.attn.v.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 23105536, "nbytes": 4096 }, "enc.9.attn.o.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 7, "offset": 23109632, "nbytes": 2097152 }, "enc.9.attn.o.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 25206784, "nbytes": 4096 }, "enc.9.attn.dist": { "dtype": "f32", "shape": [ 64, 73 ], "shard": 7, "offset": 25210880, "nbytes": 18688 }, "enc.9.conv_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 25229568, "nbytes": 4096 }, "enc.9.conv_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 25233664, "nbytes": 4096 }, "enc.9.conv.pw1.weight": { "dtype": "bf16", "shape": [ 1024, 2048 ], "shard": 7, "offset": 25237760, "nbytes": 4194304 }, "enc.9.conv.dw.weight": { "dtype": "f32", "shape": [ 1024, 31 ], "shard": 7, "offset": 29432064, "nbytes": 126976 }, "enc.9.conv.dw_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 29559040, "nbytes": 4096 }, "enc.9.conv.dw_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 29563136, "nbytes": 4096 }, "enc.9.conv.pw2.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 7, "offset": 29567232, "nbytes": 2097152 }, "enc.9.final_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 31664384, "nbytes": 4096 }, "enc.9.final_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 31668480, "nbytes": 4096 }, "enc.10.ffn1_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 31672576, "nbytes": 4096 }, "enc.10.ffn1_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 31676672, "nbytes": 4096 }, "enc.10.ffn1.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 7, "offset": 31680768, "nbytes": 8388608 }, "enc.10.ffn1.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 7, "offset": 40069376, "nbytes": 16384 }, "enc.10.ffn1.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 7, "offset": 40085760, "nbytes": 8388608 }, "enc.10.ffn1.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 48474368, "nbytes": 4096 }, "enc.10.ffn2_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 48478464, "nbytes": 4096 }, "enc.10.ffn2_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 48482560, "nbytes": 4096 }, "enc.10.ffn2.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 7, "offset": 48486656, "nbytes": 8388608 }, "enc.10.ffn2.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 7, "offset": 56875264, "nbytes": 16384 }, "enc.10.ffn2.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 7, "offset": 56891648, "nbytes": 8388608 }, "enc.10.ffn2.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 65280256, "nbytes": 4096 }, "enc.10.attn_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 65284352, "nbytes": 4096 }, "enc.10.attn_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 7, "offset": 65288448, "nbytes": 4096 }, "enc.10.attn.q.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 8, "offset": 0, "nbytes": 2097152 }, "enc.10.attn.q.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 2097152, "nbytes": 4096 }, "enc.10.attn.k.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 8, "offset": 2101248, "nbytes": 2097152 }, "enc.10.attn.k.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 4198400, "nbytes": 4096 }, "enc.10.attn.v.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 8, "offset": 4202496, "nbytes": 2097152 }, "enc.10.attn.v.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 6299648, "nbytes": 4096 }, "enc.10.attn.o.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 8, "offset": 6303744, "nbytes": 2097152 }, "enc.10.attn.o.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 8400896, "nbytes": 4096 }, "enc.10.attn.dist": { "dtype": "f32", "shape": [ 64, 73 ], "shard": 8, "offset": 8404992, "nbytes": 18688 }, "enc.10.conv_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 8423680, "nbytes": 4096 }, "enc.10.conv_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 8427776, "nbytes": 4096 }, "enc.10.conv.pw1.weight": { "dtype": "bf16", "shape": [ 1024, 2048 ], "shard": 8, "offset": 8431872, "nbytes": 4194304 }, "enc.10.conv.dw.weight": { "dtype": "f32", "shape": [ 1024, 31 ], "shard": 8, "offset": 12626176, "nbytes": 126976 }, "enc.10.conv.dw_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 12753152, "nbytes": 4096 }, "enc.10.conv.dw_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 12757248, "nbytes": 4096 }, "enc.10.conv.pw2.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 8, "offset": 12761344, "nbytes": 2097152 }, "enc.10.final_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 14858496, "nbytes": 4096 }, "enc.10.final_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 14862592, "nbytes": 4096 }, "enc.11.ffn1_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 14866688, "nbytes": 4096 }, "enc.11.ffn1_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 14870784, "nbytes": 4096 }, "enc.11.ffn1.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 8, "offset": 14874880, "nbytes": 8388608 }, "enc.11.ffn1.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 8, "offset": 23263488, "nbytes": 16384 }, "enc.11.ffn1.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 8, "offset": 23279872, "nbytes": 8388608 }, "enc.11.ffn1.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 31668480, "nbytes": 4096 }, "enc.11.ffn2_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 31672576, "nbytes": 4096 }, "enc.11.ffn2_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 31676672, "nbytes": 4096 }, "enc.11.ffn2.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 8, "offset": 31680768, "nbytes": 8388608 }, "enc.11.ffn2.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 8, "offset": 40069376, "nbytes": 16384 }, "enc.11.ffn2.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 8, "offset": 40085760, "nbytes": 8388608 }, "enc.11.ffn2.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 48474368, "nbytes": 4096 }, "enc.11.attn_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 48478464, "nbytes": 4096 }, "enc.11.attn_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 48482560, "nbytes": 4096 }, "enc.11.attn.q.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 8, "offset": 48486656, "nbytes": 2097152 }, "enc.11.attn.q.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 50583808, "nbytes": 4096 }, "enc.11.attn.k.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 8, "offset": 50587904, "nbytes": 2097152 }, "enc.11.attn.k.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 52685056, "nbytes": 4096 }, "enc.11.attn.v.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 8, "offset": 52689152, "nbytes": 2097152 }, "enc.11.attn.v.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 54786304, "nbytes": 4096 }, "enc.11.attn.o.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 8, "offset": 54790400, "nbytes": 2097152 }, "enc.11.attn.o.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 56887552, "nbytes": 4096 }, "enc.11.attn.dist": { "dtype": "f32", "shape": [ 64, 73 ], "shard": 8, "offset": 56891648, "nbytes": 18688 }, "enc.11.conv_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 56910336, "nbytes": 4096 }, "enc.11.conv_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 56914432, "nbytes": 4096 }, "enc.11.conv.pw1.weight": { "dtype": "bf16", "shape": [ 1024, 2048 ], "shard": 8, "offset": 56918528, "nbytes": 4194304 }, "enc.11.conv.dw.weight": { "dtype": "f32", "shape": [ 1024, 31 ], "shard": 8, "offset": 61112832, "nbytes": 126976 }, "enc.11.conv.dw_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 61239808, "nbytes": 4096 }, "enc.11.conv.dw_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 61243904, "nbytes": 4096 }, "enc.11.conv.pw2.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 8, "offset": 61248000, "nbytes": 2097152 }, "enc.11.final_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 63345152, "nbytes": 4096 }, "enc.11.final_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 63349248, "nbytes": 4096 }, "enc.12.ffn1_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 63353344, "nbytes": 4096 }, "enc.12.ffn1_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 8, "offset": 63357440, "nbytes": 4096 }, "enc.12.ffn1.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 9, "offset": 0, "nbytes": 8388608 }, "enc.12.ffn1.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 9, "offset": 8388608, "nbytes": 16384 }, "enc.12.ffn1.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 9, "offset": 8404992, "nbytes": 8388608 }, "enc.12.ffn1.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 16793600, "nbytes": 4096 }, "enc.12.ffn2_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 16797696, "nbytes": 4096 }, "enc.12.ffn2_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 16801792, "nbytes": 4096 }, "enc.12.ffn2.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 9, "offset": 16805888, "nbytes": 8388608 }, "enc.12.ffn2.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 9, "offset": 25194496, "nbytes": 16384 }, "enc.12.ffn2.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 9, "offset": 25210880, "nbytes": 8388608 }, "enc.12.ffn2.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 33599488, "nbytes": 4096 }, "enc.12.attn_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 33603584, "nbytes": 4096 }, "enc.12.attn_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 33607680, "nbytes": 4096 }, "enc.12.attn.q.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 9, "offset": 33611776, "nbytes": 2097152 }, "enc.12.attn.q.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 35708928, "nbytes": 4096 }, "enc.12.attn.k.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 9, "offset": 35713024, "nbytes": 2097152 }, "enc.12.attn.k.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 37810176, "nbytes": 4096 }, "enc.12.attn.v.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 9, "offset": 37814272, "nbytes": 2097152 }, "enc.12.attn.v.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 39911424, "nbytes": 4096 }, "enc.12.attn.o.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 9, "offset": 39915520, "nbytes": 2097152 }, "enc.12.attn.o.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 42012672, "nbytes": 4096 }, "enc.12.attn.dist": { "dtype": "f32", "shape": [ 64, 73 ], "shard": 9, "offset": 42016768, "nbytes": 18688 }, "enc.12.conv_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 42035456, "nbytes": 4096 }, "enc.12.conv_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 42039552, "nbytes": 4096 }, "enc.12.conv.pw1.weight": { "dtype": "bf16", "shape": [ 1024, 2048 ], "shard": 9, "offset": 42043648, "nbytes": 4194304 }, "enc.12.conv.dw.weight": { "dtype": "f32", "shape": [ 1024, 31 ], "shard": 9, "offset": 46237952, "nbytes": 126976 }, "enc.12.conv.dw_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 46364928, "nbytes": 4096 }, "enc.12.conv.dw_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 46369024, "nbytes": 4096 }, "enc.12.conv.pw2.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 9, "offset": 46373120, "nbytes": 2097152 }, "enc.12.final_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 48470272, "nbytes": 4096 }, "enc.12.final_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 48474368, "nbytes": 4096 }, "enc.13.ffn1_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 48478464, "nbytes": 4096 }, "enc.13.ffn1_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 48482560, "nbytes": 4096 }, "enc.13.ffn1.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 9, "offset": 48486656, "nbytes": 8388608 }, "enc.13.ffn1.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 9, "offset": 56875264, "nbytes": 16384 }, "enc.13.ffn1.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 9, "offset": 56891648, "nbytes": 8388608 }, "enc.13.ffn1.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 65280256, "nbytes": 4096 }, "enc.13.ffn2_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 65284352, "nbytes": 4096 }, "enc.13.ffn2_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 9, "offset": 65288448, "nbytes": 4096 }, "enc.13.ffn2.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 10, "offset": 0, "nbytes": 8388608 }, "enc.13.ffn2.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 10, "offset": 8388608, "nbytes": 16384 }, "enc.13.ffn2.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 10, "offset": 8404992, "nbytes": 8388608 }, "enc.13.ffn2.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 16793600, "nbytes": 4096 }, "enc.13.attn_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 16797696, "nbytes": 4096 }, "enc.13.attn_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 16801792, "nbytes": 4096 }, "enc.13.attn.q.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 10, "offset": 16805888, "nbytes": 2097152 }, "enc.13.attn.q.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 18903040, "nbytes": 4096 }, "enc.13.attn.k.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 10, "offset": 18907136, "nbytes": 2097152 }, "enc.13.attn.k.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 21004288, "nbytes": 4096 }, "enc.13.attn.v.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 10, "offset": 21008384, "nbytes": 2097152 }, "enc.13.attn.v.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 23105536, "nbytes": 4096 }, "enc.13.attn.o.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 10, "offset": 23109632, "nbytes": 2097152 }, "enc.13.attn.o.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 25206784, "nbytes": 4096 }, "enc.13.attn.dist": { "dtype": "f32", "shape": [ 64, 73 ], "shard": 10, "offset": 25210880, "nbytes": 18688 }, "enc.13.conv_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 25229568, "nbytes": 4096 }, "enc.13.conv_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 25233664, "nbytes": 4096 }, "enc.13.conv.pw1.weight": { "dtype": "bf16", "shape": [ 1024, 2048 ], "shard": 10, "offset": 25237760, "nbytes": 4194304 }, "enc.13.conv.dw.weight": { "dtype": "f32", "shape": [ 1024, 31 ], "shard": 10, "offset": 29432064, "nbytes": 126976 }, "enc.13.conv.dw_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 29559040, "nbytes": 4096 }, "enc.13.conv.dw_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 29563136, "nbytes": 4096 }, "enc.13.conv.pw2.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 10, "offset": 29567232, "nbytes": 2097152 }, "enc.13.final_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 31664384, "nbytes": 4096 }, "enc.13.final_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 31668480, "nbytes": 4096 }, "enc.14.ffn1_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 31672576, "nbytes": 4096 }, "enc.14.ffn1_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 31676672, "nbytes": 4096 }, "enc.14.ffn1.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 10, "offset": 31680768, "nbytes": 8388608 }, "enc.14.ffn1.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 10, "offset": 40069376, "nbytes": 16384 }, "enc.14.ffn1.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 10, "offset": 40085760, "nbytes": 8388608 }, "enc.14.ffn1.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 48474368, "nbytes": 4096 }, "enc.14.ffn2_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 48478464, "nbytes": 4096 }, "enc.14.ffn2_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 48482560, "nbytes": 4096 }, "enc.14.ffn2.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 10, "offset": 48486656, "nbytes": 8388608 }, "enc.14.ffn2.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 10, "offset": 56875264, "nbytes": 16384 }, "enc.14.ffn2.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 10, "offset": 56891648, "nbytes": 8388608 }, "enc.14.ffn2.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 65280256, "nbytes": 4096 }, "enc.14.attn_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 65284352, "nbytes": 4096 }, "enc.14.attn_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 10, "offset": 65288448, "nbytes": 4096 }, "enc.14.attn.q.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 11, "offset": 0, "nbytes": 2097152 }, "enc.14.attn.q.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 2097152, "nbytes": 4096 }, "enc.14.attn.k.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 11, "offset": 2101248, "nbytes": 2097152 }, "enc.14.attn.k.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 4198400, "nbytes": 4096 }, "enc.14.attn.v.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 11, "offset": 4202496, "nbytes": 2097152 }, "enc.14.attn.v.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 6299648, "nbytes": 4096 }, "enc.14.attn.o.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 11, "offset": 6303744, "nbytes": 2097152 }, "enc.14.attn.o.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 8400896, "nbytes": 4096 }, "enc.14.attn.dist": { "dtype": "f32", "shape": [ 64, 73 ], "shard": 11, "offset": 8404992, "nbytes": 18688 }, "enc.14.conv_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 8423680, "nbytes": 4096 }, "enc.14.conv_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 8427776, "nbytes": 4096 }, "enc.14.conv.pw1.weight": { "dtype": "bf16", "shape": [ 1024, 2048 ], "shard": 11, "offset": 8431872, "nbytes": 4194304 }, "enc.14.conv.dw.weight": { "dtype": "f32", "shape": [ 1024, 31 ], "shard": 11, "offset": 12626176, "nbytes": 126976 }, "enc.14.conv.dw_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 12753152, "nbytes": 4096 }, "enc.14.conv.dw_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 12757248, "nbytes": 4096 }, "enc.14.conv.pw2.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 11, "offset": 12761344, "nbytes": 2097152 }, "enc.14.final_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 14858496, "nbytes": 4096 }, "enc.14.final_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 14862592, "nbytes": 4096 }, "enc.15.ffn1_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 14866688, "nbytes": 4096 }, "enc.15.ffn1_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 14870784, "nbytes": 4096 }, "enc.15.ffn1.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 11, "offset": 14874880, "nbytes": 8388608 }, "enc.15.ffn1.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 11, "offset": 23263488, "nbytes": 16384 }, "enc.15.ffn1.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 11, "offset": 23279872, "nbytes": 8388608 }, "enc.15.ffn1.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 31668480, "nbytes": 4096 }, "enc.15.ffn2_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 31672576, "nbytes": 4096 }, "enc.15.ffn2_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 31676672, "nbytes": 4096 }, "enc.15.ffn2.up.weight": { "dtype": "bf16", "shape": [ 1024, 4096 ], "shard": 11, "offset": 31680768, "nbytes": 8388608 }, "enc.15.ffn2.up.bias": { "dtype": "f32", "shape": [ 4096 ], "shard": 11, "offset": 40069376, "nbytes": 16384 }, "enc.15.ffn2.down.weight": { "dtype": "bf16", "shape": [ 4096, 1024 ], "shard": 11, "offset": 40085760, "nbytes": 8388608 }, "enc.15.ffn2.down.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 48474368, "nbytes": 4096 }, "enc.15.attn_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 48478464, "nbytes": 4096 }, "enc.15.attn_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 48482560, "nbytes": 4096 }, "enc.15.attn.q.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 11, "offset": 48486656, "nbytes": 2097152 }, "enc.15.attn.q.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 50583808, "nbytes": 4096 }, "enc.15.attn.k.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 11, "offset": 50587904, "nbytes": 2097152 }, "enc.15.attn.k.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 52685056, "nbytes": 4096 }, "enc.15.attn.v.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 11, "offset": 52689152, "nbytes": 2097152 }, "enc.15.attn.v.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 54786304, "nbytes": 4096 }, "enc.15.attn.o.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 11, "offset": 54790400, "nbytes": 2097152 }, "enc.15.attn.o.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 56887552, "nbytes": 4096 }, "enc.15.attn.dist": { "dtype": "f32", "shape": [ 64, 73 ], "shard": 11, "offset": 56891648, "nbytes": 18688 }, "enc.15.conv_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 56910336, "nbytes": 4096 }, "enc.15.conv_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 56914432, "nbytes": 4096 }, "enc.15.conv.pw1.weight": { "dtype": "bf16", "shape": [ 1024, 2048 ], "shard": 11, "offset": 56918528, "nbytes": 4194304 }, "enc.15.conv.dw.weight": { "dtype": "f32", "shape": [ 1024, 31 ], "shard": 11, "offset": 61112832, "nbytes": 126976 }, "enc.15.conv.dw_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 61239808, "nbytes": 4096 }, "enc.15.conv.dw_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 61243904, "nbytes": 4096 }, "enc.15.conv.pw2.weight": { "dtype": "bf16", "shape": [ 1024, 1024 ], "shard": 11, "offset": 61248000, "nbytes": 2097152 }, "enc.15.final_ln.weight": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 63345152, "nbytes": 4096 }, "enc.15.final_ln.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 11, "offset": 63349248, "nbytes": 4096 } } }