{ "format": "zonos3-web-v1", "model": "zac-full_mr", "meta": { "general.architecture": "mossdual_istft", "general.name": "ZONOS3 Mini MR-iSTFT decoder (full_mr, step 50000)", "sample_rate": 44100, "internal_sample_rate": 44160, "frame_rate": 15, "hop": 2940, "internal_hop": 2944, "n_codebooks": 10, "codebook_size": 4096, "latent_dim": 1024, "semantic_dim": 768, "sem_head.depth": 4, "head.depth": 8, "head.hidden": 2048, "head.hop": 184, "head.fir_kernel": 127, "head.step": 50000, "head.n_fft": [ 736, 2208 ], "head.max_log_magnitude": 4.605170249938965, "headroom_db": 3.0, "head.variant": "full_mr", "ln_eps": 9.999999747378752e-06, "convnext_eps": 9.999999974752427e-07, "stage.in_dim": [ 1024, 640, 384, 384, 384 ], "stage.out_dim": [ 1280, 768, 768, 768, 0 ], "stage.d_model": [ 1280, 768, 768, 768, 768 ], "stage.n_head": [ 20, 12, 12, 12, 12 ], "stage.n_ff": [ 5120, 3072, 3072, 3072, 3072 ], "stage.context": [ 150, 300, 480, 480, 480 ], "stage.n_layer": [ 32, 12, 12, 12, 12 ], "stage.patch": [ 2, 2, 2, 2, 1 ], "resample.orig": 736, "resample.new": 735, "resample.width": 17 }, "shards": [ "shard-000.bin", "shard-001.bin", "shard-002.bin", "shard-003.bin", "shard-004.bin", "shard-005.bin", "shard-006.bin", "shard-007.bin", "shard-008.bin", "shard-009.bin", "shard-010.bin", "shard-011.bin", "shard-012.bin", "shard-013.bin", "shard-014.bin", "shard-015.bin", "shard-016.bin", "shard-017.bin", "shard-018.bin", "shard-019.bin" ], "tensors": { "sem.table": { "dtype": "f16", "shape": [ 768, 4096 ], "shard": 0, "offset": 0, "nbytes": 6291456 }, "res.tables": { "dtype": "f16", "shape": [ 1024, 36864 ], "shard": 1, "offset": 0, "nbytes": 75497472 }, "res.biases": { "dtype": "f32", "shape": [ 9216 ], "shard": 2, "offset": 0, "nbytes": 36864 }, "sem.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 2, "offset": 36864, "nbytes": 3072 }, "res.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 39936, "nbytes": 4096 }, "sem_head.0.dw.weight": { "dtype": "f32", "shape": [ 768, 7 ], "shard": 2, "offset": 44032, "nbytes": 21504 }, "sem_head.0.dw.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 2, "offset": 65536, "nbytes": 3072 }, "sem_head.0.norm.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 2, "offset": 68608, "nbytes": 3072 }, "sem_head.0.norm.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 2, "offset": 71680, "nbytes": 3072 }, "sem_head.0.pw1.weight": { "dtype": "q8", "shape": [ 768, 2048 ], "shard": 2, "offset": 74752, "nbytes": 1572864, "scales_offset": 1647616, "scales_nbytes": 196608 }, "sem_head.0.pw1.bias": { "dtype": "f32", "shape": [ 2048 ], "shard": 2, "offset": 1844224, "nbytes": 8192 }, "sem_head.0.pw2.weight": { "dtype": "q8", "shape": [ 2048, 768 ], "shard": 2, "offset": 1852416, "nbytes": 1572864, "scales_offset": 3425280, "scales_nbytes": 196608 }, "sem_head.0.pw2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 2, "offset": 3621888, "nbytes": 3072 }, "sem_head.1.dw.weight": { "dtype": "f32", "shape": [ 768, 7 ], "shard": 2, "offset": 3624960, "nbytes": 21504 }, "sem_head.1.dw.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 2, "offset": 3646464, "nbytes": 3072 }, "sem_head.1.norm.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 2, "offset": 3649536, "nbytes": 3072 }, "sem_head.1.norm.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 2, "offset": 3652608, "nbytes": 3072 }, "sem_head.1.pw1.weight": { "dtype": "q8", "shape": [ 768, 2048 ], "shard": 2, "offset": 3655680, "nbytes": 1572864, "scales_offset": 5228544, "scales_nbytes": 196608 }, "sem_head.1.pw1.bias": { "dtype": "f32", "shape": [ 2048 ], "shard": 2, "offset": 5425152, "nbytes": 8192 }, "sem_head.1.pw2.weight": { "dtype": "q8", "shape": [ 2048, 768 ], "shard": 2, "offset": 5433344, "nbytes": 1572864, "scales_offset": 7006208, "scales_nbytes": 196608 }, "sem_head.1.pw2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 2, "offset": 7202816, "nbytes": 3072 }, "sem_head.2.dw.weight": { "dtype": "f32", "shape": [ 768, 7 ], "shard": 2, "offset": 7205888, "nbytes": 21504 }, "sem_head.2.dw.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 2, "offset": 7227392, "nbytes": 3072 }, "sem_head.2.norm.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 2, "offset": 7230464, "nbytes": 3072 }, "sem_head.2.norm.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 2, "offset": 7233536, "nbytes": 3072 }, "sem_head.2.pw1.weight": { "dtype": "q8", "shape": [ 768, 2048 ], "shard": 2, "offset": 7236608, "nbytes": 1572864, "scales_offset": 8809472, "scales_nbytes": 196608 }, "sem_head.2.pw1.bias": { "dtype": "f32", "shape": [ 2048 ], "shard": 2, "offset": 9006080, "nbytes": 8192 }, "sem_head.2.pw2.weight": { "dtype": "q8", "shape": [ 2048, 768 ], "shard": 2, "offset": 9014272, "nbytes": 1572864, "scales_offset": 10587136, "scales_nbytes": 196608 }, "sem_head.2.pw2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 2, "offset": 10783744, "nbytes": 3072 }, "sem_head.3.dw.weight": { "dtype": "f32", "shape": [ 768, 7 ], "shard": 2, "offset": 10786816, "nbytes": 21504 }, "sem_head.3.dw.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 2, "offset": 10808320, "nbytes": 3072 }, "sem_head.3.norm.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 2, "offset": 10811392, "nbytes": 3072 }, "sem_head.3.norm.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 2, "offset": 10814464, "nbytes": 3072 }, "sem_head.3.pw1.weight": { "dtype": "q8", "shape": [ 768, 2048 ], "shard": 2, "offset": 10817536, "nbytes": 1572864, "scales_offset": 12390400, "scales_nbytes": 196608 }, "sem_head.3.pw1.bias": { "dtype": "f32", "shape": [ 2048 ], "shard": 2, "offset": 12587008, "nbytes": 8192 }, "sem_head.3.pw2.weight": { "dtype": "q8", "shape": [ 2048, 768 ], "shard": 2, "offset": 12595200, "nbytes": 1572864, "scales_offset": 14168064, "scales_nbytes": 196608 }, "sem_head.3.pw2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 2, "offset": 14364672, "nbytes": 3072 }, "sem_head.out.weight": { "dtype": "q8", "shape": [ 768, 1024 ], "shard": 2, "offset": 14367744, "nbytes": 786432, "scales_offset": 15154176, "scales_nbytes": 98304 }, "sem_head.out.bias": { "dtype": "f32", "shape": [ 1024 ], "shard": 2, "offset": 15252480, "nbytes": 4096 }, "stage.0.in.weight": { "dtype": "q8", "shape": [ 1024, 1280 ], "shard": 2, "offset": 15256576, "nbytes": 1310720, "scales_offset": 16567296, "scales_nbytes": 163840 }, "stage.0.out.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 2, "offset": 16731136, "nbytes": 1638400, "scales_offset": 18369536, "scales_nbytes": 204800 }, "stage.0.l0.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 2, "offset": 18574336, "nbytes": 5120 }, "stage.0.l0.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 2, "offset": 18579456, "nbytes": 5120 }, "stage.0.l0.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 2, "offset": 18584576, "nbytes": 4915200, "scales_offset": 23499776, "scales_nbytes": 614400 }, "stage.0.l0.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 2, "offset": 24114176, "nbytes": 1638400, "scales_offset": 25752576, "scales_nbytes": 204800 }, "stage.0.l0.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 2, "offset": 25957376, "nbytes": 5120 }, "stage.0.l0.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 2, "offset": 25962496, "nbytes": 5120 }, "stage.0.l0.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 2, "offset": 25967616, "nbytes": 5120 }, "stage.0.l0.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 2, "offset": 25972736, "nbytes": 6553600, "scales_offset": 32526336, "scales_nbytes": 819200 }, "stage.0.l0.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 2, "offset": 33345536, "nbytes": 6553600, "scales_offset": 39899136, "scales_nbytes": 819200 }, "stage.0.l0.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 2, "offset": 40718336, "nbytes": 5120 }, "stage.0.l1.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 2, "offset": 40723456, "nbytes": 5120 }, "stage.0.l1.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 2, "offset": 40728576, "nbytes": 5120 }, "stage.0.l1.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 2, "offset": 40733696, "nbytes": 4915200, "scales_offset": 45648896, "scales_nbytes": 614400 }, "stage.0.l1.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 2, "offset": 46263296, "nbytes": 1638400, "scales_offset": 47901696, "scales_nbytes": 204800 }, "stage.0.l1.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 2, "offset": 48106496, "nbytes": 5120 }, "stage.0.l1.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 2, "offset": 48111616, "nbytes": 5120 }, "stage.0.l1.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 2, "offset": 48116736, "nbytes": 5120 }, "stage.0.l1.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 2, "offset": 48121856, "nbytes": 6553600, "scales_offset": 54675456, "scales_nbytes": 819200 }, "stage.0.l1.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 2, "offset": 55494656, "nbytes": 6553600, "scales_offset": 62048256, "scales_nbytes": 819200 }, "stage.0.l1.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 2, "offset": 62867456, "nbytes": 5120 }, "stage.0.l2.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 2, "offset": 62872576, "nbytes": 5120 }, "stage.0.l2.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 2, "offset": 62877696, "nbytes": 5120 }, "stage.0.l2.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 3, "offset": 0, "nbytes": 4915200, "scales_offset": 4915200, "scales_nbytes": 614400 }, "stage.0.l2.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 3, "offset": 5529600, "nbytes": 1638400, "scales_offset": 7168000, "scales_nbytes": 204800 }, "stage.0.l2.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 3, "offset": 7372800, "nbytes": 5120 }, "stage.0.l2.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 3, "offset": 7377920, "nbytes": 5120 }, "stage.0.l2.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 3, "offset": 7383040, "nbytes": 5120 }, "stage.0.l2.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 3, "offset": 7388160, "nbytes": 6553600, "scales_offset": 13941760, "scales_nbytes": 819200 }, "stage.0.l2.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 3, "offset": 14760960, "nbytes": 6553600, "scales_offset": 21314560, "scales_nbytes": 819200 }, "stage.0.l2.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 3, "offset": 22133760, "nbytes": 5120 }, "stage.0.l3.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 3, "offset": 22138880, "nbytes": 5120 }, "stage.0.l3.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 3, "offset": 22144000, "nbytes": 5120 }, "stage.0.l3.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 3, "offset": 22149120, "nbytes": 4915200, "scales_offset": 27064320, "scales_nbytes": 614400 }, "stage.0.l3.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 3, "offset": 27678720, "nbytes": 1638400, "scales_offset": 29317120, "scales_nbytes": 204800 }, "stage.0.l3.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 3, "offset": 29521920, "nbytes": 5120 }, "stage.0.l3.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 3, "offset": 29527040, "nbytes": 5120 }, "stage.0.l3.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 3, "offset": 29532160, "nbytes": 5120 }, "stage.0.l3.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 3, "offset": 29537280, "nbytes": 6553600, "scales_offset": 36090880, "scales_nbytes": 819200 }, "stage.0.l3.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 3, "offset": 36910080, "nbytes": 6553600, "scales_offset": 43463680, "scales_nbytes": 819200 }, "stage.0.l3.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 3, "offset": 44282880, "nbytes": 5120 }, "stage.0.l4.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 3, "offset": 44288000, "nbytes": 5120 }, "stage.0.l4.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 3, "offset": 44293120, "nbytes": 5120 }, "stage.0.l4.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 3, "offset": 44298240, "nbytes": 4915200, "scales_offset": 49213440, "scales_nbytes": 614400 }, "stage.0.l4.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 3, "offset": 49827840, "nbytes": 1638400, "scales_offset": 51466240, "scales_nbytes": 204800 }, "stage.0.l4.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 3, "offset": 51671040, "nbytes": 5120 }, "stage.0.l4.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 3, "offset": 51676160, "nbytes": 5120 }, "stage.0.l4.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 3, "offset": 51681280, "nbytes": 5120 }, "stage.0.l4.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 3, "offset": 51686400, "nbytes": 6553600, "scales_offset": 58240000, "scales_nbytes": 819200 }, "stage.0.l4.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 3, "offset": 59059200, "nbytes": 6553600, "scales_offset": 65612800, "scales_nbytes": 819200 }, "stage.0.l4.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 3, "offset": 66432000, "nbytes": 5120 }, "stage.0.l5.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 3, "offset": 66437120, "nbytes": 5120 }, "stage.0.l5.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 3, "offset": 66442240, "nbytes": 5120 }, "stage.0.l5.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 4, "offset": 0, "nbytes": 4915200, "scales_offset": 4915200, "scales_nbytes": 614400 }, "stage.0.l5.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 4, "offset": 5529600, "nbytes": 1638400, "scales_offset": 7168000, "scales_nbytes": 204800 }, "stage.0.l5.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 4, "offset": 7372800, "nbytes": 5120 }, "stage.0.l5.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 4, "offset": 7377920, "nbytes": 5120 }, "stage.0.l5.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 4, "offset": 7383040, "nbytes": 5120 }, "stage.0.l5.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 4, "offset": 7388160, "nbytes": 6553600, "scales_offset": 13941760, "scales_nbytes": 819200 }, "stage.0.l5.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 4, "offset": 14760960, "nbytes": 6553600, "scales_offset": 21314560, "scales_nbytes": 819200 }, "stage.0.l5.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 4, "offset": 22133760, "nbytes": 5120 }, "stage.0.l6.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 4, "offset": 22138880, "nbytes": 5120 }, "stage.0.l6.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 4, "offset": 22144000, "nbytes": 5120 }, "stage.0.l6.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 4, "offset": 22149120, "nbytes": 4915200, "scales_offset": 27064320, "scales_nbytes": 614400 }, "stage.0.l6.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 4, "offset": 27678720, "nbytes": 1638400, "scales_offset": 29317120, "scales_nbytes": 204800 }, "stage.0.l6.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 4, "offset": 29521920, "nbytes": 5120 }, "stage.0.l6.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 4, "offset": 29527040, "nbytes": 5120 }, "stage.0.l6.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 4, "offset": 29532160, "nbytes": 5120 }, "stage.0.l6.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 4, "offset": 29537280, "nbytes": 6553600, "scales_offset": 36090880, "scales_nbytes": 819200 }, "stage.0.l6.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 4, "offset": 36910080, "nbytes": 6553600, "scales_offset": 43463680, "scales_nbytes": 819200 }, "stage.0.l6.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 4, "offset": 44282880, "nbytes": 5120 }, "stage.0.l7.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 4, "offset": 44288000, "nbytes": 5120 }, "stage.0.l7.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 4, "offset": 44293120, "nbytes": 5120 }, "stage.0.l7.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 4, "offset": 44298240, "nbytes": 4915200, "scales_offset": 49213440, "scales_nbytes": 614400 }, "stage.0.l7.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 4, "offset": 49827840, "nbytes": 1638400, "scales_offset": 51466240, "scales_nbytes": 204800 }, "stage.0.l7.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 4, "offset": 51671040, "nbytes": 5120 }, "stage.0.l7.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 4, "offset": 51676160, "nbytes": 5120 }, "stage.0.l7.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 4, "offset": 51681280, "nbytes": 5120 }, "stage.0.l7.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 4, "offset": 51686400, "nbytes": 6553600, "scales_offset": 58240000, "scales_nbytes": 819200 }, "stage.0.l7.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 4, "offset": 59059200, "nbytes": 6553600, "scales_offset": 65612800, "scales_nbytes": 819200 }, "stage.0.l7.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 4, "offset": 66432000, "nbytes": 5120 }, "stage.0.l8.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 4, "offset": 66437120, "nbytes": 5120 }, "stage.0.l8.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 4, "offset": 66442240, "nbytes": 5120 }, "stage.0.l8.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 5, "offset": 0, "nbytes": 4915200, "scales_offset": 4915200, "scales_nbytes": 614400 }, "stage.0.l8.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 5, "offset": 5529600, "nbytes": 1638400, "scales_offset": 7168000, "scales_nbytes": 204800 }, "stage.0.l8.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 5, "offset": 7372800, "nbytes": 5120 }, "stage.0.l8.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 5, "offset": 7377920, "nbytes": 5120 }, "stage.0.l8.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 5, "offset": 7383040, "nbytes": 5120 }, "stage.0.l8.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 5, "offset": 7388160, "nbytes": 6553600, "scales_offset": 13941760, "scales_nbytes": 819200 }, "stage.0.l8.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 5, "offset": 14760960, "nbytes": 6553600, "scales_offset": 21314560, "scales_nbytes": 819200 }, "stage.0.l8.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 5, "offset": 22133760, "nbytes": 5120 }, "stage.0.l9.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 5, "offset": 22138880, "nbytes": 5120 }, "stage.0.l9.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 5, "offset": 22144000, "nbytes": 5120 }, "stage.0.l9.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 5, "offset": 22149120, "nbytes": 4915200, "scales_offset": 27064320, "scales_nbytes": 614400 }, "stage.0.l9.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 5, "offset": 27678720, "nbytes": 1638400, "scales_offset": 29317120, "scales_nbytes": 204800 }, "stage.0.l9.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 5, "offset": 29521920, "nbytes": 5120 }, "stage.0.l9.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 5, "offset": 29527040, "nbytes": 5120 }, "stage.0.l9.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 5, "offset": 29532160, "nbytes": 5120 }, "stage.0.l9.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 5, "offset": 29537280, "nbytes": 6553600, "scales_offset": 36090880, "scales_nbytes": 819200 }, "stage.0.l9.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 5, "offset": 36910080, "nbytes": 6553600, "scales_offset": 43463680, "scales_nbytes": 819200 }, "stage.0.l9.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 5, "offset": 44282880, "nbytes": 5120 }, "stage.0.l10.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 5, "offset": 44288000, "nbytes": 5120 }, "stage.0.l10.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 5, "offset": 44293120, "nbytes": 5120 }, "stage.0.l10.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 5, "offset": 44298240, "nbytes": 4915200, "scales_offset": 49213440, "scales_nbytes": 614400 }, "stage.0.l10.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 5, "offset": 49827840, "nbytes": 1638400, "scales_offset": 51466240, "scales_nbytes": 204800 }, "stage.0.l10.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 5, "offset": 51671040, "nbytes": 5120 }, "stage.0.l10.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 5, "offset": 51676160, "nbytes": 5120 }, "stage.0.l10.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 5, "offset": 51681280, "nbytes": 5120 }, "stage.0.l10.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 5, "offset": 51686400, "nbytes": 6553600, "scales_offset": 58240000, "scales_nbytes": 819200 }, "stage.0.l10.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 5, "offset": 59059200, "nbytes": 6553600, "scales_offset": 65612800, "scales_nbytes": 819200 }, "stage.0.l10.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 5, "offset": 66432000, "nbytes": 5120 }, "stage.0.l11.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 5, "offset": 66437120, "nbytes": 5120 }, "stage.0.l11.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 5, "offset": 66442240, "nbytes": 5120 }, "stage.0.l11.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 6, "offset": 0, "nbytes": 4915200, "scales_offset": 4915200, "scales_nbytes": 614400 }, "stage.0.l11.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 6, "offset": 5529600, "nbytes": 1638400, "scales_offset": 7168000, "scales_nbytes": 204800 }, "stage.0.l11.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 6, "offset": 7372800, "nbytes": 5120 }, "stage.0.l11.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 6, "offset": 7377920, "nbytes": 5120 }, "stage.0.l11.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 6, "offset": 7383040, "nbytes": 5120 }, "stage.0.l11.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 6, "offset": 7388160, "nbytes": 6553600, "scales_offset": 13941760, "scales_nbytes": 819200 }, "stage.0.l11.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 6, "offset": 14760960, "nbytes": 6553600, "scales_offset": 21314560, "scales_nbytes": 819200 }, "stage.0.l11.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 6, "offset": 22133760, "nbytes": 5120 }, "stage.0.l12.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 6, "offset": 22138880, "nbytes": 5120 }, "stage.0.l12.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 6, "offset": 22144000, "nbytes": 5120 }, "stage.0.l12.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 6, "offset": 22149120, "nbytes": 4915200, "scales_offset": 27064320, "scales_nbytes": 614400 }, "stage.0.l12.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 6, "offset": 27678720, "nbytes": 1638400, "scales_offset": 29317120, "scales_nbytes": 204800 }, "stage.0.l12.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 6, "offset": 29521920, "nbytes": 5120 }, "stage.0.l12.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 6, "offset": 29527040, "nbytes": 5120 }, "stage.0.l12.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 6, "offset": 29532160, "nbytes": 5120 }, "stage.0.l12.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 6, "offset": 29537280, "nbytes": 6553600, "scales_offset": 36090880, "scales_nbytes": 819200 }, "stage.0.l12.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 6, "offset": 36910080, "nbytes": 6553600, "scales_offset": 43463680, "scales_nbytes": 819200 }, "stage.0.l12.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 6, "offset": 44282880, "nbytes": 5120 }, "stage.0.l13.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 6, "offset": 44288000, "nbytes": 5120 }, "stage.0.l13.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 6, "offset": 44293120, "nbytes": 5120 }, "stage.0.l13.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 6, "offset": 44298240, "nbytes": 4915200, "scales_offset": 49213440, "scales_nbytes": 614400 }, "stage.0.l13.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 6, "offset": 49827840, "nbytes": 1638400, "scales_offset": 51466240, "scales_nbytes": 204800 }, "stage.0.l13.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 6, "offset": 51671040, "nbytes": 5120 }, "stage.0.l13.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 6, "offset": 51676160, "nbytes": 5120 }, "stage.0.l13.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 6, "offset": 51681280, "nbytes": 5120 }, "stage.0.l13.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 6, "offset": 51686400, "nbytes": 6553600, "scales_offset": 58240000, "scales_nbytes": 819200 }, "stage.0.l13.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 6, "offset": 59059200, "nbytes": 6553600, "scales_offset": 65612800, "scales_nbytes": 819200 }, "stage.0.l13.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 6, "offset": 66432000, "nbytes": 5120 }, "stage.0.l14.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 6, "offset": 66437120, "nbytes": 5120 }, "stage.0.l14.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 6, "offset": 66442240, "nbytes": 5120 }, "stage.0.l14.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 7, "offset": 0, "nbytes": 4915200, "scales_offset": 4915200, "scales_nbytes": 614400 }, "stage.0.l14.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 7, "offset": 5529600, "nbytes": 1638400, "scales_offset": 7168000, "scales_nbytes": 204800 }, "stage.0.l14.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 7, "offset": 7372800, "nbytes": 5120 }, "stage.0.l14.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 7, "offset": 7377920, "nbytes": 5120 }, "stage.0.l14.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 7, "offset": 7383040, "nbytes": 5120 }, "stage.0.l14.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 7, "offset": 7388160, "nbytes": 6553600, "scales_offset": 13941760, "scales_nbytes": 819200 }, "stage.0.l14.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 7, "offset": 14760960, "nbytes": 6553600, "scales_offset": 21314560, "scales_nbytes": 819200 }, "stage.0.l14.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 7, "offset": 22133760, "nbytes": 5120 }, "stage.0.l15.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 7, "offset": 22138880, "nbytes": 5120 }, "stage.0.l15.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 7, "offset": 22144000, "nbytes": 5120 }, "stage.0.l15.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 7, "offset": 22149120, "nbytes": 4915200, "scales_offset": 27064320, "scales_nbytes": 614400 }, "stage.0.l15.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 7, "offset": 27678720, "nbytes": 1638400, "scales_offset": 29317120, "scales_nbytes": 204800 }, "stage.0.l15.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 7, "offset": 29521920, "nbytes": 5120 }, "stage.0.l15.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 7, "offset": 29527040, "nbytes": 5120 }, "stage.0.l15.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 7, "offset": 29532160, "nbytes": 5120 }, "stage.0.l15.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 7, "offset": 29537280, "nbytes": 6553600, "scales_offset": 36090880, "scales_nbytes": 819200 }, "stage.0.l15.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 7, "offset": 36910080, "nbytes": 6553600, "scales_offset": 43463680, "scales_nbytes": 819200 }, "stage.0.l15.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 7, "offset": 44282880, "nbytes": 5120 }, "stage.0.l16.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 7, "offset": 44288000, "nbytes": 5120 }, "stage.0.l16.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 7, "offset": 44293120, "nbytes": 5120 }, "stage.0.l16.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 7, "offset": 44298240, "nbytes": 4915200, "scales_offset": 49213440, "scales_nbytes": 614400 }, "stage.0.l16.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 7, "offset": 49827840, "nbytes": 1638400, "scales_offset": 51466240, "scales_nbytes": 204800 }, "stage.0.l16.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 7, "offset": 51671040, "nbytes": 5120 }, "stage.0.l16.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 7, "offset": 51676160, "nbytes": 5120 }, "stage.0.l16.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 7, "offset": 51681280, "nbytes": 5120 }, "stage.0.l16.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 7, "offset": 51686400, "nbytes": 6553600, "scales_offset": 58240000, "scales_nbytes": 819200 }, "stage.0.l16.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 7, "offset": 59059200, "nbytes": 6553600, "scales_offset": 65612800, "scales_nbytes": 819200 }, "stage.0.l16.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 7, "offset": 66432000, "nbytes": 5120 }, "stage.0.l17.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 7, "offset": 66437120, "nbytes": 5120 }, "stage.0.l17.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 7, "offset": 66442240, "nbytes": 5120 }, "stage.0.l17.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 8, "offset": 0, "nbytes": 4915200, "scales_offset": 4915200, "scales_nbytes": 614400 }, "stage.0.l17.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 8, "offset": 5529600, "nbytes": 1638400, "scales_offset": 7168000, "scales_nbytes": 204800 }, "stage.0.l17.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 8, "offset": 7372800, "nbytes": 5120 }, "stage.0.l17.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 8, "offset": 7377920, "nbytes": 5120 }, "stage.0.l17.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 8, "offset": 7383040, "nbytes": 5120 }, "stage.0.l17.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 8, "offset": 7388160, "nbytes": 6553600, "scales_offset": 13941760, "scales_nbytes": 819200 }, "stage.0.l17.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 8, "offset": 14760960, "nbytes": 6553600, "scales_offset": 21314560, "scales_nbytes": 819200 }, "stage.0.l17.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 8, "offset": 22133760, "nbytes": 5120 }, "stage.0.l18.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 8, "offset": 22138880, "nbytes": 5120 }, "stage.0.l18.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 8, "offset": 22144000, "nbytes": 5120 }, "stage.0.l18.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 8, "offset": 22149120, "nbytes": 4915200, "scales_offset": 27064320, "scales_nbytes": 614400 }, "stage.0.l18.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 8, "offset": 27678720, "nbytes": 1638400, "scales_offset": 29317120, "scales_nbytes": 204800 }, "stage.0.l18.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 8, "offset": 29521920, "nbytes": 5120 }, "stage.0.l18.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 8, "offset": 29527040, "nbytes": 5120 }, "stage.0.l18.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 8, "offset": 29532160, "nbytes": 5120 }, "stage.0.l18.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 8, "offset": 29537280, "nbytes": 6553600, "scales_offset": 36090880, "scales_nbytes": 819200 }, "stage.0.l18.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 8, "offset": 36910080, "nbytes": 6553600, "scales_offset": 43463680, "scales_nbytes": 819200 }, "stage.0.l18.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 8, "offset": 44282880, "nbytes": 5120 }, "stage.0.l19.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 8, "offset": 44288000, "nbytes": 5120 }, "stage.0.l19.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 8, "offset": 44293120, "nbytes": 5120 }, "stage.0.l19.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 8, "offset": 44298240, "nbytes": 4915200, "scales_offset": 49213440, "scales_nbytes": 614400 }, "stage.0.l19.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 8, "offset": 49827840, "nbytes": 1638400, "scales_offset": 51466240, "scales_nbytes": 204800 }, "stage.0.l19.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 8, "offset": 51671040, "nbytes": 5120 }, "stage.0.l19.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 8, "offset": 51676160, "nbytes": 5120 }, "stage.0.l19.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 8, "offset": 51681280, "nbytes": 5120 }, "stage.0.l19.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 8, "offset": 51686400, "nbytes": 6553600, "scales_offset": 58240000, "scales_nbytes": 819200 }, "stage.0.l19.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 8, "offset": 59059200, "nbytes": 6553600, "scales_offset": 65612800, "scales_nbytes": 819200 }, "stage.0.l19.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 8, "offset": 66432000, "nbytes": 5120 }, "stage.0.l20.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 8, "offset": 66437120, "nbytes": 5120 }, "stage.0.l20.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 8, "offset": 66442240, "nbytes": 5120 }, "stage.0.l20.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 9, "offset": 0, "nbytes": 4915200, "scales_offset": 4915200, "scales_nbytes": 614400 }, "stage.0.l20.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 9, "offset": 5529600, "nbytes": 1638400, "scales_offset": 7168000, "scales_nbytes": 204800 }, "stage.0.l20.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 9, "offset": 7372800, "nbytes": 5120 }, "stage.0.l20.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 9, "offset": 7377920, "nbytes": 5120 }, "stage.0.l20.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 9, "offset": 7383040, "nbytes": 5120 }, "stage.0.l20.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 9, "offset": 7388160, "nbytes": 6553600, "scales_offset": 13941760, "scales_nbytes": 819200 }, "stage.0.l20.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 9, "offset": 14760960, "nbytes": 6553600, "scales_offset": 21314560, "scales_nbytes": 819200 }, "stage.0.l20.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 9, "offset": 22133760, "nbytes": 5120 }, "stage.0.l21.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 9, "offset": 22138880, "nbytes": 5120 }, "stage.0.l21.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 9, "offset": 22144000, "nbytes": 5120 }, "stage.0.l21.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 9, "offset": 22149120, "nbytes": 4915200, "scales_offset": 27064320, "scales_nbytes": 614400 }, "stage.0.l21.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 9, "offset": 27678720, "nbytes": 1638400, "scales_offset": 29317120, "scales_nbytes": 204800 }, "stage.0.l21.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 9, "offset": 29521920, "nbytes": 5120 }, "stage.0.l21.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 9, "offset": 29527040, "nbytes": 5120 }, "stage.0.l21.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 9, "offset": 29532160, "nbytes": 5120 }, "stage.0.l21.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 9, "offset": 29537280, "nbytes": 6553600, "scales_offset": 36090880, "scales_nbytes": 819200 }, "stage.0.l21.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 9, "offset": 36910080, "nbytes": 6553600, "scales_offset": 43463680, "scales_nbytes": 819200 }, "stage.0.l21.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 9, "offset": 44282880, "nbytes": 5120 }, "stage.0.l22.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 9, "offset": 44288000, "nbytes": 5120 }, "stage.0.l22.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 9, "offset": 44293120, "nbytes": 5120 }, "stage.0.l22.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 9, "offset": 44298240, "nbytes": 4915200, "scales_offset": 49213440, "scales_nbytes": 614400 }, "stage.0.l22.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 9, "offset": 49827840, "nbytes": 1638400, "scales_offset": 51466240, "scales_nbytes": 204800 }, "stage.0.l22.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 9, "offset": 51671040, "nbytes": 5120 }, "stage.0.l22.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 9, "offset": 51676160, "nbytes": 5120 }, "stage.0.l22.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 9, "offset": 51681280, "nbytes": 5120 }, "stage.0.l22.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 9, "offset": 51686400, "nbytes": 6553600, "scales_offset": 58240000, "scales_nbytes": 819200 }, "stage.0.l22.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 9, "offset": 59059200, "nbytes": 6553600, "scales_offset": 65612800, "scales_nbytes": 819200 }, "stage.0.l22.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 9, "offset": 66432000, "nbytes": 5120 }, "stage.0.l23.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 9, "offset": 66437120, "nbytes": 5120 }, "stage.0.l23.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 9, "offset": 66442240, "nbytes": 5120 }, "stage.0.l23.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 10, "offset": 0, "nbytes": 4915200, "scales_offset": 4915200, "scales_nbytes": 614400 }, "stage.0.l23.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 10, "offset": 5529600, "nbytes": 1638400, "scales_offset": 7168000, "scales_nbytes": 204800 }, "stage.0.l23.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 10, "offset": 7372800, "nbytes": 5120 }, "stage.0.l23.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 10, "offset": 7377920, "nbytes": 5120 }, "stage.0.l23.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 10, "offset": 7383040, "nbytes": 5120 }, "stage.0.l23.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 10, "offset": 7388160, "nbytes": 6553600, "scales_offset": 13941760, "scales_nbytes": 819200 }, "stage.0.l23.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 10, "offset": 14760960, "nbytes": 6553600, "scales_offset": 21314560, "scales_nbytes": 819200 }, "stage.0.l23.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 10, "offset": 22133760, "nbytes": 5120 }, "stage.0.l24.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 10, "offset": 22138880, "nbytes": 5120 }, "stage.0.l24.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 10, "offset": 22144000, "nbytes": 5120 }, "stage.0.l24.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 10, "offset": 22149120, "nbytes": 4915200, "scales_offset": 27064320, "scales_nbytes": 614400 }, "stage.0.l24.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 10, "offset": 27678720, "nbytes": 1638400, "scales_offset": 29317120, "scales_nbytes": 204800 }, "stage.0.l24.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 10, "offset": 29521920, "nbytes": 5120 }, "stage.0.l24.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 10, "offset": 29527040, "nbytes": 5120 }, "stage.0.l24.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 10, "offset": 29532160, "nbytes": 5120 }, "stage.0.l24.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 10, "offset": 29537280, "nbytes": 6553600, "scales_offset": 36090880, "scales_nbytes": 819200 }, "stage.0.l24.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 10, "offset": 36910080, "nbytes": 6553600, "scales_offset": 43463680, "scales_nbytes": 819200 }, "stage.0.l24.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 10, "offset": 44282880, "nbytes": 5120 }, "stage.0.l25.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 10, "offset": 44288000, "nbytes": 5120 }, "stage.0.l25.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 10, "offset": 44293120, "nbytes": 5120 }, "stage.0.l25.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 10, "offset": 44298240, "nbytes": 4915200, "scales_offset": 49213440, "scales_nbytes": 614400 }, "stage.0.l25.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 10, "offset": 49827840, "nbytes": 1638400, "scales_offset": 51466240, "scales_nbytes": 204800 }, "stage.0.l25.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 10, "offset": 51671040, "nbytes": 5120 }, "stage.0.l25.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 10, "offset": 51676160, "nbytes": 5120 }, "stage.0.l25.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 10, "offset": 51681280, "nbytes": 5120 }, "stage.0.l25.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 10, "offset": 51686400, "nbytes": 6553600, "scales_offset": 58240000, "scales_nbytes": 819200 }, "stage.0.l25.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 10, "offset": 59059200, "nbytes": 6553600, "scales_offset": 65612800, "scales_nbytes": 819200 }, "stage.0.l25.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 10, "offset": 66432000, "nbytes": 5120 }, "stage.0.l26.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 10, "offset": 66437120, "nbytes": 5120 }, "stage.0.l26.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 10, "offset": 66442240, "nbytes": 5120 }, "stage.0.l26.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 11, "offset": 0, "nbytes": 4915200, "scales_offset": 4915200, "scales_nbytes": 614400 }, "stage.0.l26.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 11, "offset": 5529600, "nbytes": 1638400, "scales_offset": 7168000, "scales_nbytes": 204800 }, "stage.0.l26.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 11, "offset": 7372800, "nbytes": 5120 }, "stage.0.l26.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 11, "offset": 7377920, "nbytes": 5120 }, "stage.0.l26.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 11, "offset": 7383040, "nbytes": 5120 }, "stage.0.l26.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 11, "offset": 7388160, "nbytes": 6553600, "scales_offset": 13941760, "scales_nbytes": 819200 }, "stage.0.l26.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 11, "offset": 14760960, "nbytes": 6553600, "scales_offset": 21314560, "scales_nbytes": 819200 }, "stage.0.l26.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 11, "offset": 22133760, "nbytes": 5120 }, "stage.0.l27.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 11, "offset": 22138880, "nbytes": 5120 }, "stage.0.l27.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 11, "offset": 22144000, "nbytes": 5120 }, "stage.0.l27.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 11, "offset": 22149120, "nbytes": 4915200, "scales_offset": 27064320, "scales_nbytes": 614400 }, "stage.0.l27.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 11, "offset": 27678720, "nbytes": 1638400, "scales_offset": 29317120, "scales_nbytes": 204800 }, "stage.0.l27.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 11, "offset": 29521920, "nbytes": 5120 }, "stage.0.l27.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 11, "offset": 29527040, "nbytes": 5120 }, "stage.0.l27.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 11, "offset": 29532160, "nbytes": 5120 }, "stage.0.l27.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 11, "offset": 29537280, "nbytes": 6553600, "scales_offset": 36090880, "scales_nbytes": 819200 }, "stage.0.l27.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 11, "offset": 36910080, "nbytes": 6553600, "scales_offset": 43463680, "scales_nbytes": 819200 }, "stage.0.l27.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 11, "offset": 44282880, "nbytes": 5120 }, "stage.0.l28.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 11, "offset": 44288000, "nbytes": 5120 }, "stage.0.l28.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 11, "offset": 44293120, "nbytes": 5120 }, "stage.0.l28.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 11, "offset": 44298240, "nbytes": 4915200, "scales_offset": 49213440, "scales_nbytes": 614400 }, "stage.0.l28.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 11, "offset": 49827840, "nbytes": 1638400, "scales_offset": 51466240, "scales_nbytes": 204800 }, "stage.0.l28.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 11, "offset": 51671040, "nbytes": 5120 }, "stage.0.l28.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 11, "offset": 51676160, "nbytes": 5120 }, "stage.0.l28.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 11, "offset": 51681280, "nbytes": 5120 }, "stage.0.l28.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 11, "offset": 51686400, "nbytes": 6553600, "scales_offset": 58240000, "scales_nbytes": 819200 }, "stage.0.l28.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 11, "offset": 59059200, "nbytes": 6553600, "scales_offset": 65612800, "scales_nbytes": 819200 }, "stage.0.l28.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 11, "offset": 66432000, "nbytes": 5120 }, "stage.0.l29.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 11, "offset": 66437120, "nbytes": 5120 }, "stage.0.l29.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 11, "offset": 66442240, "nbytes": 5120 }, "stage.0.l29.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 12, "offset": 0, "nbytes": 4915200, "scales_offset": 4915200, "scales_nbytes": 614400 }, "stage.0.l29.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 12, "offset": 5529600, "nbytes": 1638400, "scales_offset": 7168000, "scales_nbytes": 204800 }, "stage.0.l29.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 12, "offset": 7372800, "nbytes": 5120 }, "stage.0.l29.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 12, "offset": 7377920, "nbytes": 5120 }, "stage.0.l29.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 12, "offset": 7383040, "nbytes": 5120 }, "stage.0.l29.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 12, "offset": 7388160, "nbytes": 6553600, "scales_offset": 13941760, "scales_nbytes": 819200 }, "stage.0.l29.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 12, "offset": 14760960, "nbytes": 6553600, "scales_offset": 21314560, "scales_nbytes": 819200 }, "stage.0.l29.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 12, "offset": 22133760, "nbytes": 5120 }, "stage.0.l30.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 12, "offset": 22138880, "nbytes": 5120 }, "stage.0.l30.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 12, "offset": 22144000, "nbytes": 5120 }, "stage.0.l30.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 12, "offset": 22149120, "nbytes": 4915200, "scales_offset": 27064320, "scales_nbytes": 614400 }, "stage.0.l30.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 12, "offset": 27678720, "nbytes": 1638400, "scales_offset": 29317120, "scales_nbytes": 204800 }, "stage.0.l30.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 12, "offset": 29521920, "nbytes": 5120 }, "stage.0.l30.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 12, "offset": 29527040, "nbytes": 5120 }, "stage.0.l30.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 12, "offset": 29532160, "nbytes": 5120 }, "stage.0.l30.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 12, "offset": 29537280, "nbytes": 6553600, "scales_offset": 36090880, "scales_nbytes": 819200 }, "stage.0.l30.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 12, "offset": 36910080, "nbytes": 6553600, "scales_offset": 43463680, "scales_nbytes": 819200 }, "stage.0.l30.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 12, "offset": 44282880, "nbytes": 5120 }, "stage.0.l31.norm1.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 12, "offset": 44288000, "nbytes": 5120 }, "stage.0.l31.norm1.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 12, "offset": 44293120, "nbytes": 5120 }, "stage.0.l31.qkv.weight": { "dtype": "q8", "shape": [ 1280, 3840 ], "shard": 12, "offset": 44298240, "nbytes": 4915200, "scales_offset": 49213440, "scales_nbytes": 614400 }, "stage.0.l31.wo.weight": { "dtype": "q8", "shape": [ 1280, 1280 ], "shard": 12, "offset": 49827840, "nbytes": 1638400, "scales_offset": 51466240, "scales_nbytes": 204800 }, "stage.0.l31.attn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 12, "offset": 51671040, "nbytes": 5120 }, "stage.0.l31.norm2.weight": { "dtype": "f32", "shape": [ 1280 ], "shard": 12, "offset": 51676160, "nbytes": 5120 }, "stage.0.l31.norm2.bias": { "dtype": "f32", "shape": [ 1280 ], "shard": 12, "offset": 51681280, "nbytes": 5120 }, "stage.0.l31.ffn1.weight": { "dtype": "q8", "shape": [ 1280, 5120 ], "shard": 12, "offset": 51686400, "nbytes": 6553600, "scales_offset": 58240000, "scales_nbytes": 819200 }, "stage.0.l31.ffn2.weight": { "dtype": "q8", "shape": [ 5120, 1280 ], "shard": 12, "offset": 59059200, "nbytes": 6553600, "scales_offset": 65612800, "scales_nbytes": 819200 }, "stage.0.l31.ffn_scale": { "dtype": "f32", "shape": [ 1280 ], "shard": 12, "offset": 66432000, "nbytes": 5120 }, "stage.1.in.weight": { "dtype": "q8", "shape": [ 640, 768 ], "shard": 12, "offset": 66437120, "nbytes": 491520, "scales_offset": 66928640, "scales_nbytes": 61440 }, "stage.1.out.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 13, "offset": 0, "nbytes": 589824, "scales_offset": 589824, "scales_nbytes": 73728 }, "stage.1.l0.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 663552, "nbytes": 3072 }, "stage.1.l0.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 666624, "nbytes": 3072 }, "stage.1.l0.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 13, "offset": 669696, "nbytes": 1769472, "scales_offset": 2439168, "scales_nbytes": 221184 }, "stage.1.l0.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 13, "offset": 2660352, "nbytes": 589824, "scales_offset": 3250176, "scales_nbytes": 73728 }, "stage.1.l0.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 3323904, "nbytes": 3072 }, "stage.1.l0.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 3326976, "nbytes": 3072 }, "stage.1.l0.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 3330048, "nbytes": 3072 }, "stage.1.l0.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 13, "offset": 3333120, "nbytes": 2359296, "scales_offset": 5692416, "scales_nbytes": 294912 }, "stage.1.l0.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 13, "offset": 5987328, "nbytes": 2359296, "scales_offset": 8346624, "scales_nbytes": 294912 }, "stage.1.l0.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 8641536, "nbytes": 3072 }, "stage.1.l1.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 8644608, "nbytes": 3072 }, "stage.1.l1.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 8647680, "nbytes": 3072 }, "stage.1.l1.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 13, "offset": 8650752, "nbytes": 1769472, "scales_offset": 10420224, "scales_nbytes": 221184 }, "stage.1.l1.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 13, "offset": 10641408, "nbytes": 589824, "scales_offset": 11231232, "scales_nbytes": 73728 }, "stage.1.l1.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 11304960, "nbytes": 3072 }, "stage.1.l1.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 11308032, "nbytes": 3072 }, "stage.1.l1.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 11311104, "nbytes": 3072 }, "stage.1.l1.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 13, "offset": 11314176, "nbytes": 2359296, "scales_offset": 13673472, "scales_nbytes": 294912 }, "stage.1.l1.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 13, "offset": 13968384, "nbytes": 2359296, "scales_offset": 16327680, "scales_nbytes": 294912 }, "stage.1.l1.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 16622592, "nbytes": 3072 }, "stage.1.l2.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 16625664, "nbytes": 3072 }, "stage.1.l2.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 16628736, "nbytes": 3072 }, "stage.1.l2.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 13, "offset": 16631808, "nbytes": 1769472, "scales_offset": 18401280, "scales_nbytes": 221184 }, "stage.1.l2.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 13, "offset": 18622464, "nbytes": 589824, "scales_offset": 19212288, "scales_nbytes": 73728 }, "stage.1.l2.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 19286016, "nbytes": 3072 }, "stage.1.l2.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 19289088, "nbytes": 3072 }, "stage.1.l2.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 19292160, "nbytes": 3072 }, "stage.1.l2.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 13, "offset": 19295232, "nbytes": 2359296, "scales_offset": 21654528, "scales_nbytes": 294912 }, "stage.1.l2.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 13, "offset": 21949440, "nbytes": 2359296, "scales_offset": 24308736, "scales_nbytes": 294912 }, "stage.1.l2.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 24603648, "nbytes": 3072 }, "stage.1.l3.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 24606720, "nbytes": 3072 }, "stage.1.l3.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 24609792, "nbytes": 3072 }, "stage.1.l3.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 13, "offset": 24612864, "nbytes": 1769472, "scales_offset": 26382336, "scales_nbytes": 221184 }, "stage.1.l3.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 13, "offset": 26603520, "nbytes": 589824, "scales_offset": 27193344, "scales_nbytes": 73728 }, "stage.1.l3.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 27267072, "nbytes": 3072 }, "stage.1.l3.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 27270144, "nbytes": 3072 }, "stage.1.l3.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 27273216, "nbytes": 3072 }, "stage.1.l3.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 13, "offset": 27276288, "nbytes": 2359296, "scales_offset": 29635584, "scales_nbytes": 294912 }, "stage.1.l3.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 13, "offset": 29930496, "nbytes": 2359296, "scales_offset": 32289792, "scales_nbytes": 294912 }, "stage.1.l3.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 32584704, "nbytes": 3072 }, "stage.1.l4.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 32587776, "nbytes": 3072 }, "stage.1.l4.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 32590848, "nbytes": 3072 }, "stage.1.l4.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 13, "offset": 32593920, "nbytes": 1769472, "scales_offset": 34363392, "scales_nbytes": 221184 }, "stage.1.l4.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 13, "offset": 34584576, "nbytes": 589824, "scales_offset": 35174400, "scales_nbytes": 73728 }, "stage.1.l4.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 35248128, "nbytes": 3072 }, "stage.1.l4.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 35251200, "nbytes": 3072 }, "stage.1.l4.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 35254272, "nbytes": 3072 }, "stage.1.l4.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 13, "offset": 35257344, "nbytes": 2359296, "scales_offset": 37616640, "scales_nbytes": 294912 }, "stage.1.l4.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 13, "offset": 37911552, "nbytes": 2359296, "scales_offset": 40270848, "scales_nbytes": 294912 }, "stage.1.l4.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 40565760, "nbytes": 3072 }, "stage.1.l5.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 40568832, "nbytes": 3072 }, "stage.1.l5.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 40571904, "nbytes": 3072 }, "stage.1.l5.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 13, "offset": 40574976, "nbytes": 1769472, "scales_offset": 42344448, "scales_nbytes": 221184 }, "stage.1.l5.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 13, "offset": 42565632, "nbytes": 589824, "scales_offset": 43155456, "scales_nbytes": 73728 }, "stage.1.l5.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 43229184, "nbytes": 3072 }, "stage.1.l5.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 43232256, "nbytes": 3072 }, "stage.1.l5.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 43235328, "nbytes": 3072 }, "stage.1.l5.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 13, "offset": 43238400, "nbytes": 2359296, "scales_offset": 45597696, "scales_nbytes": 294912 }, "stage.1.l5.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 13, "offset": 45892608, "nbytes": 2359296, "scales_offset": 48251904, "scales_nbytes": 294912 }, "stage.1.l5.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 48546816, "nbytes": 3072 }, "stage.1.l6.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 48549888, "nbytes": 3072 }, "stage.1.l6.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 48552960, "nbytes": 3072 }, "stage.1.l6.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 13, "offset": 48556032, "nbytes": 1769472, "scales_offset": 50325504, "scales_nbytes": 221184 }, "stage.1.l6.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 13, "offset": 50546688, "nbytes": 589824, "scales_offset": 51136512, "scales_nbytes": 73728 }, "stage.1.l6.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 51210240, "nbytes": 3072 }, "stage.1.l6.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 51213312, "nbytes": 3072 }, "stage.1.l6.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 51216384, "nbytes": 3072 }, "stage.1.l6.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 13, "offset": 51219456, "nbytes": 2359296, "scales_offset": 53578752, "scales_nbytes": 294912 }, "stage.1.l6.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 13, "offset": 53873664, "nbytes": 2359296, "scales_offset": 56232960, "scales_nbytes": 294912 }, "stage.1.l6.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 56527872, "nbytes": 3072 }, "stage.1.l7.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 56530944, "nbytes": 3072 }, "stage.1.l7.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 56534016, "nbytes": 3072 }, "stage.1.l7.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 13, "offset": 56537088, "nbytes": 1769472, "scales_offset": 58306560, "scales_nbytes": 221184 }, "stage.1.l7.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 13, "offset": 58527744, "nbytes": 589824, "scales_offset": 59117568, "scales_nbytes": 73728 }, "stage.1.l7.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 59191296, "nbytes": 3072 }, "stage.1.l7.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 59194368, "nbytes": 3072 }, "stage.1.l7.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 59197440, "nbytes": 3072 }, "stage.1.l7.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 13, "offset": 59200512, "nbytes": 2359296, "scales_offset": 61559808, "scales_nbytes": 294912 }, "stage.1.l7.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 13, "offset": 61854720, "nbytes": 2359296, "scales_offset": 64214016, "scales_nbytes": 294912 }, "stage.1.l7.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 64508928, "nbytes": 3072 }, "stage.1.l8.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 64512000, "nbytes": 3072 }, "stage.1.l8.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 13, "offset": 64515072, "nbytes": 3072 }, "stage.1.l8.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 13, "offset": 64518144, "nbytes": 1769472, "scales_offset": 66287616, "scales_nbytes": 221184 }, "stage.1.l8.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 14, "offset": 0, "nbytes": 589824, "scales_offset": 589824, "scales_nbytes": 73728 }, "stage.1.l8.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 663552, "nbytes": 3072 }, "stage.1.l8.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 666624, "nbytes": 3072 }, "stage.1.l8.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 669696, "nbytes": 3072 }, "stage.1.l8.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 14, "offset": 672768, "nbytes": 2359296, "scales_offset": 3032064, "scales_nbytes": 294912 }, "stage.1.l8.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 14, "offset": 3326976, "nbytes": 2359296, "scales_offset": 5686272, "scales_nbytes": 294912 }, "stage.1.l8.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 5981184, "nbytes": 3072 }, "stage.1.l9.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 5984256, "nbytes": 3072 }, "stage.1.l9.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 5987328, "nbytes": 3072 }, "stage.1.l9.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 14, "offset": 5990400, "nbytes": 1769472, "scales_offset": 7759872, "scales_nbytes": 221184 }, "stage.1.l9.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 14, "offset": 7981056, "nbytes": 589824, "scales_offset": 8570880, "scales_nbytes": 73728 }, "stage.1.l9.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 8644608, "nbytes": 3072 }, "stage.1.l9.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 8647680, "nbytes": 3072 }, "stage.1.l9.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 8650752, "nbytes": 3072 }, "stage.1.l9.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 14, "offset": 8653824, "nbytes": 2359296, "scales_offset": 11013120, "scales_nbytes": 294912 }, "stage.1.l9.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 14, "offset": 11308032, "nbytes": 2359296, "scales_offset": 13667328, "scales_nbytes": 294912 }, "stage.1.l9.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 13962240, "nbytes": 3072 }, "stage.1.l10.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 13965312, "nbytes": 3072 }, "stage.1.l10.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 13968384, "nbytes": 3072 }, "stage.1.l10.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 14, "offset": 13971456, "nbytes": 1769472, "scales_offset": 15740928, "scales_nbytes": 221184 }, "stage.1.l10.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 14, "offset": 15962112, "nbytes": 589824, "scales_offset": 16551936, "scales_nbytes": 73728 }, "stage.1.l10.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 16625664, "nbytes": 3072 }, "stage.1.l10.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 16628736, "nbytes": 3072 }, "stage.1.l10.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 16631808, "nbytes": 3072 }, "stage.1.l10.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 14, "offset": 16634880, "nbytes": 2359296, "scales_offset": 18994176, "scales_nbytes": 294912 }, "stage.1.l10.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 14, "offset": 19289088, "nbytes": 2359296, "scales_offset": 21648384, "scales_nbytes": 294912 }, "stage.1.l10.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 21943296, "nbytes": 3072 }, "stage.1.l11.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 21946368, "nbytes": 3072 }, "stage.1.l11.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 21949440, "nbytes": 3072 }, "stage.1.l11.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 14, "offset": 21952512, "nbytes": 1769472, "scales_offset": 23721984, "scales_nbytes": 221184 }, "stage.1.l11.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 14, "offset": 23943168, "nbytes": 589824, "scales_offset": 24532992, "scales_nbytes": 73728 }, "stage.1.l11.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 24606720, "nbytes": 3072 }, "stage.1.l11.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 24609792, "nbytes": 3072 }, "stage.1.l11.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 24612864, "nbytes": 3072 }, "stage.1.l11.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 14, "offset": 24615936, "nbytes": 2359296, "scales_offset": 26975232, "scales_nbytes": 294912 }, "stage.1.l11.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 14, "offset": 27270144, "nbytes": 2359296, "scales_offset": 29629440, "scales_nbytes": 294912 }, "stage.1.l11.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 29924352, "nbytes": 3072 }, "stage.2.in.weight": { "dtype": "q8", "shape": [ 384, 768 ], "shard": 14, "offset": 29927424, "nbytes": 294912, "scales_offset": 30222336, "scales_nbytes": 36864 }, "stage.2.out.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 14, "offset": 30259200, "nbytes": 589824, "scales_offset": 30849024, "scales_nbytes": 73728 }, "stage.2.l0.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 30922752, "nbytes": 3072 }, "stage.2.l0.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 30925824, "nbytes": 3072 }, "stage.2.l0.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 14, "offset": 30928896, "nbytes": 1769472, "scales_offset": 32698368, "scales_nbytes": 221184 }, "stage.2.l0.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 14, "offset": 32919552, "nbytes": 589824, "scales_offset": 33509376, "scales_nbytes": 73728 }, "stage.2.l0.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 33583104, "nbytes": 3072 }, "stage.2.l0.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 33586176, "nbytes": 3072 }, "stage.2.l0.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 33589248, "nbytes": 3072 }, "stage.2.l0.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 14, "offset": 33592320, "nbytes": 2359296, "scales_offset": 35951616, "scales_nbytes": 294912 }, "stage.2.l0.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 14, "offset": 36246528, "nbytes": 2359296, "scales_offset": 38605824, "scales_nbytes": 294912 }, "stage.2.l0.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 38900736, "nbytes": 3072 }, "stage.2.l1.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 38903808, "nbytes": 3072 }, "stage.2.l1.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 38906880, "nbytes": 3072 }, "stage.2.l1.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 14, "offset": 38909952, "nbytes": 1769472, "scales_offset": 40679424, "scales_nbytes": 221184 }, "stage.2.l1.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 14, "offset": 40900608, "nbytes": 589824, "scales_offset": 41490432, "scales_nbytes": 73728 }, "stage.2.l1.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 41564160, "nbytes": 3072 }, "stage.2.l1.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 41567232, "nbytes": 3072 }, "stage.2.l1.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 41570304, "nbytes": 3072 }, "stage.2.l1.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 14, "offset": 41573376, "nbytes": 2359296, "scales_offset": 43932672, "scales_nbytes": 294912 }, "stage.2.l1.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 14, "offset": 44227584, "nbytes": 2359296, "scales_offset": 46586880, "scales_nbytes": 294912 }, "stage.2.l1.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 46881792, "nbytes": 3072 }, "stage.2.l2.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 46884864, "nbytes": 3072 }, "stage.2.l2.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 46887936, "nbytes": 3072 }, "stage.2.l2.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 14, "offset": 46891008, "nbytes": 1769472, "scales_offset": 48660480, "scales_nbytes": 221184 }, "stage.2.l2.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 14, "offset": 48881664, "nbytes": 589824, "scales_offset": 49471488, "scales_nbytes": 73728 }, "stage.2.l2.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 49545216, "nbytes": 3072 }, "stage.2.l2.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 49548288, "nbytes": 3072 }, "stage.2.l2.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 49551360, "nbytes": 3072 }, "stage.2.l2.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 14, "offset": 49554432, "nbytes": 2359296, "scales_offset": 51913728, "scales_nbytes": 294912 }, "stage.2.l2.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 14, "offset": 52208640, "nbytes": 2359296, "scales_offset": 54567936, "scales_nbytes": 294912 }, "stage.2.l2.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 54862848, "nbytes": 3072 }, "stage.2.l3.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 54865920, "nbytes": 3072 }, "stage.2.l3.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 54868992, "nbytes": 3072 }, "stage.2.l3.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 14, "offset": 54872064, "nbytes": 1769472, "scales_offset": 56641536, "scales_nbytes": 221184 }, "stage.2.l3.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 14, "offset": 56862720, "nbytes": 589824, "scales_offset": 57452544, "scales_nbytes": 73728 }, "stage.2.l3.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 57526272, "nbytes": 3072 }, "stage.2.l3.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 57529344, "nbytes": 3072 }, "stage.2.l3.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 57532416, "nbytes": 3072 }, "stage.2.l3.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 14, "offset": 57535488, "nbytes": 2359296, "scales_offset": 59894784, "scales_nbytes": 294912 }, "stage.2.l3.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 14, "offset": 60189696, "nbytes": 2359296, "scales_offset": 62548992, "scales_nbytes": 294912 }, "stage.2.l3.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 62843904, "nbytes": 3072 }, "stage.2.l4.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 62846976, "nbytes": 3072 }, "stage.2.l4.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 62850048, "nbytes": 3072 }, "stage.2.l4.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 14, "offset": 62853120, "nbytes": 1769472, "scales_offset": 64622592, "scales_nbytes": 221184 }, "stage.2.l4.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 14, "offset": 64843776, "nbytes": 589824, "scales_offset": 65433600, "scales_nbytes": 73728 }, "stage.2.l4.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 65507328, "nbytes": 3072 }, "stage.2.l4.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 65510400, "nbytes": 3072 }, "stage.2.l4.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 14, "offset": 65513472, "nbytes": 3072 }, "stage.2.l4.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 15, "offset": 0, "nbytes": 2359296, "scales_offset": 2359296, "scales_nbytes": 294912 }, "stage.2.l4.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 15, "offset": 2654208, "nbytes": 2359296, "scales_offset": 5013504, "scales_nbytes": 294912 }, "stage.2.l4.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 5308416, "nbytes": 3072 }, "stage.2.l5.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 5311488, "nbytes": 3072 }, "stage.2.l5.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 5314560, "nbytes": 3072 }, "stage.2.l5.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 15, "offset": 5317632, "nbytes": 1769472, "scales_offset": 7087104, "scales_nbytes": 221184 }, "stage.2.l5.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 15, "offset": 7308288, "nbytes": 589824, "scales_offset": 7898112, "scales_nbytes": 73728 }, "stage.2.l5.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 7971840, "nbytes": 3072 }, "stage.2.l5.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 7974912, "nbytes": 3072 }, "stage.2.l5.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 7977984, "nbytes": 3072 }, "stage.2.l5.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 15, "offset": 7981056, "nbytes": 2359296, "scales_offset": 10340352, "scales_nbytes": 294912 }, "stage.2.l5.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 15, "offset": 10635264, "nbytes": 2359296, "scales_offset": 12994560, "scales_nbytes": 294912 }, "stage.2.l5.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 13289472, "nbytes": 3072 }, "stage.2.l6.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 13292544, "nbytes": 3072 }, "stage.2.l6.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 13295616, "nbytes": 3072 }, "stage.2.l6.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 15, "offset": 13298688, "nbytes": 1769472, "scales_offset": 15068160, "scales_nbytes": 221184 }, "stage.2.l6.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 15, "offset": 15289344, "nbytes": 589824, "scales_offset": 15879168, "scales_nbytes": 73728 }, "stage.2.l6.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 15952896, "nbytes": 3072 }, "stage.2.l6.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 15955968, "nbytes": 3072 }, "stage.2.l6.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 15959040, "nbytes": 3072 }, "stage.2.l6.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 15, "offset": 15962112, "nbytes": 2359296, "scales_offset": 18321408, "scales_nbytes": 294912 }, "stage.2.l6.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 15, "offset": 18616320, "nbytes": 2359296, "scales_offset": 20975616, "scales_nbytes": 294912 }, "stage.2.l6.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 21270528, "nbytes": 3072 }, "stage.2.l7.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 21273600, "nbytes": 3072 }, "stage.2.l7.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 21276672, "nbytes": 3072 }, "stage.2.l7.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 15, "offset": 21279744, "nbytes": 1769472, "scales_offset": 23049216, "scales_nbytes": 221184 }, "stage.2.l7.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 15, "offset": 23270400, "nbytes": 589824, "scales_offset": 23860224, "scales_nbytes": 73728 }, "stage.2.l7.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 23933952, "nbytes": 3072 }, "stage.2.l7.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 23937024, "nbytes": 3072 }, "stage.2.l7.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 23940096, "nbytes": 3072 }, "stage.2.l7.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 15, "offset": 23943168, "nbytes": 2359296, "scales_offset": 26302464, "scales_nbytes": 294912 }, "stage.2.l7.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 15, "offset": 26597376, "nbytes": 2359296, "scales_offset": 28956672, "scales_nbytes": 294912 }, "stage.2.l7.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 29251584, "nbytes": 3072 }, "stage.2.l8.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 29254656, "nbytes": 3072 }, "stage.2.l8.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 29257728, "nbytes": 3072 }, "stage.2.l8.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 15, "offset": 29260800, "nbytes": 1769472, "scales_offset": 31030272, "scales_nbytes": 221184 }, "stage.2.l8.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 15, "offset": 31251456, "nbytes": 589824, "scales_offset": 31841280, "scales_nbytes": 73728 }, "stage.2.l8.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 31915008, "nbytes": 3072 }, "stage.2.l8.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 31918080, "nbytes": 3072 }, "stage.2.l8.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 31921152, "nbytes": 3072 }, "stage.2.l8.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 15, "offset": 31924224, "nbytes": 2359296, "scales_offset": 34283520, "scales_nbytes": 294912 }, "stage.2.l8.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 15, "offset": 34578432, "nbytes": 2359296, "scales_offset": 36937728, "scales_nbytes": 294912 }, "stage.2.l8.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 37232640, "nbytes": 3072 }, "stage.2.l9.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 37235712, "nbytes": 3072 }, "stage.2.l9.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 37238784, "nbytes": 3072 }, "stage.2.l9.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 15, "offset": 37241856, "nbytes": 1769472, "scales_offset": 39011328, "scales_nbytes": 221184 }, "stage.2.l9.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 15, "offset": 39232512, "nbytes": 589824, "scales_offset": 39822336, "scales_nbytes": 73728 }, "stage.2.l9.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 39896064, "nbytes": 3072 }, "stage.2.l9.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 39899136, "nbytes": 3072 }, "stage.2.l9.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 39902208, "nbytes": 3072 }, "stage.2.l9.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 15, "offset": 39905280, "nbytes": 2359296, "scales_offset": 42264576, "scales_nbytes": 294912 }, "stage.2.l9.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 15, "offset": 42559488, "nbytes": 2359296, "scales_offset": 44918784, "scales_nbytes": 294912 }, "stage.2.l9.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 45213696, "nbytes": 3072 }, "stage.2.l10.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 45216768, "nbytes": 3072 }, "stage.2.l10.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 45219840, "nbytes": 3072 }, "stage.2.l10.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 15, "offset": 45222912, "nbytes": 1769472, "scales_offset": 46992384, "scales_nbytes": 221184 }, "stage.2.l10.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 15, "offset": 47213568, "nbytes": 589824, "scales_offset": 47803392, "scales_nbytes": 73728 }, "stage.2.l10.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 47877120, "nbytes": 3072 }, "stage.2.l10.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 47880192, "nbytes": 3072 }, "stage.2.l10.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 47883264, "nbytes": 3072 }, "stage.2.l10.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 15, "offset": 47886336, "nbytes": 2359296, "scales_offset": 50245632, "scales_nbytes": 294912 }, "stage.2.l10.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 15, "offset": 50540544, "nbytes": 2359296, "scales_offset": 52899840, "scales_nbytes": 294912 }, "stage.2.l10.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 53194752, "nbytes": 3072 }, "stage.2.l11.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 53197824, "nbytes": 3072 }, "stage.2.l11.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 53200896, "nbytes": 3072 }, "stage.2.l11.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 15, "offset": 53203968, "nbytes": 1769472, "scales_offset": 54973440, "scales_nbytes": 221184 }, "stage.2.l11.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 15, "offset": 55194624, "nbytes": 589824, "scales_offset": 55784448, "scales_nbytes": 73728 }, "stage.2.l11.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 55858176, "nbytes": 3072 }, "stage.2.l11.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 55861248, "nbytes": 3072 }, "stage.2.l11.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 55864320, "nbytes": 3072 }, "stage.2.l11.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 15, "offset": 55867392, "nbytes": 2359296, "scales_offset": 58226688, "scales_nbytes": 294912 }, "stage.2.l11.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 15, "offset": 58521600, "nbytes": 2359296, "scales_offset": 60880896, "scales_nbytes": 294912 }, "stage.2.l11.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 61175808, "nbytes": 3072 }, "stage.3.in.weight": { "dtype": "q8", "shape": [ 384, 768 ], "shard": 15, "offset": 61178880, "nbytes": 294912, "scales_offset": 61473792, "scales_nbytes": 36864 }, "stage.3.out.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 15, "offset": 61510656, "nbytes": 589824, "scales_offset": 62100480, "scales_nbytes": 73728 }, "stage.3.l0.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 62174208, "nbytes": 3072 }, "stage.3.l0.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 62177280, "nbytes": 3072 }, "stage.3.l0.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 15, "offset": 62180352, "nbytes": 1769472, "scales_offset": 63949824, "scales_nbytes": 221184 }, "stage.3.l0.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 15, "offset": 64171008, "nbytes": 589824, "scales_offset": 64760832, "scales_nbytes": 73728 }, "stage.3.l0.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 64834560, "nbytes": 3072 }, "stage.3.l0.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 64837632, "nbytes": 3072 }, "stage.3.l0.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 15, "offset": 64840704, "nbytes": 3072 }, "stage.3.l0.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 16, "offset": 0, "nbytes": 2359296, "scales_offset": 2359296, "scales_nbytes": 294912 }, "stage.3.l0.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 16, "offset": 2654208, "nbytes": 2359296, "scales_offset": 5013504, "scales_nbytes": 294912 }, "stage.3.l0.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 5308416, "nbytes": 3072 }, "stage.3.l1.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 5311488, "nbytes": 3072 }, "stage.3.l1.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 5314560, "nbytes": 3072 }, "stage.3.l1.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 16, "offset": 5317632, "nbytes": 1769472, "scales_offset": 7087104, "scales_nbytes": 221184 }, "stage.3.l1.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 16, "offset": 7308288, "nbytes": 589824, "scales_offset": 7898112, "scales_nbytes": 73728 }, "stage.3.l1.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 7971840, "nbytes": 3072 }, "stage.3.l1.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 7974912, "nbytes": 3072 }, "stage.3.l1.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 7977984, "nbytes": 3072 }, "stage.3.l1.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 16, "offset": 7981056, "nbytes": 2359296, "scales_offset": 10340352, "scales_nbytes": 294912 }, "stage.3.l1.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 16, "offset": 10635264, "nbytes": 2359296, "scales_offset": 12994560, "scales_nbytes": 294912 }, "stage.3.l1.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 13289472, "nbytes": 3072 }, "stage.3.l2.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 13292544, "nbytes": 3072 }, "stage.3.l2.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 13295616, "nbytes": 3072 }, "stage.3.l2.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 16, "offset": 13298688, "nbytes": 1769472, "scales_offset": 15068160, "scales_nbytes": 221184 }, "stage.3.l2.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 16, "offset": 15289344, "nbytes": 589824, "scales_offset": 15879168, "scales_nbytes": 73728 }, "stage.3.l2.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 15952896, "nbytes": 3072 }, "stage.3.l2.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 15955968, "nbytes": 3072 }, "stage.3.l2.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 15959040, "nbytes": 3072 }, "stage.3.l2.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 16, "offset": 15962112, "nbytes": 2359296, "scales_offset": 18321408, "scales_nbytes": 294912 }, "stage.3.l2.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 16, "offset": 18616320, "nbytes": 2359296, "scales_offset": 20975616, "scales_nbytes": 294912 }, "stage.3.l2.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 21270528, "nbytes": 3072 }, "stage.3.l3.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 21273600, "nbytes": 3072 }, "stage.3.l3.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 21276672, "nbytes": 3072 }, "stage.3.l3.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 16, "offset": 21279744, "nbytes": 1769472, "scales_offset": 23049216, "scales_nbytes": 221184 }, "stage.3.l3.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 16, "offset": 23270400, "nbytes": 589824, "scales_offset": 23860224, "scales_nbytes": 73728 }, "stage.3.l3.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 23933952, "nbytes": 3072 }, "stage.3.l3.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 23937024, "nbytes": 3072 }, "stage.3.l3.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 23940096, "nbytes": 3072 }, "stage.3.l3.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 16, "offset": 23943168, "nbytes": 2359296, "scales_offset": 26302464, "scales_nbytes": 294912 }, "stage.3.l3.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 16, "offset": 26597376, "nbytes": 2359296, "scales_offset": 28956672, "scales_nbytes": 294912 }, "stage.3.l3.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 29251584, "nbytes": 3072 }, "stage.3.l4.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 29254656, "nbytes": 3072 }, "stage.3.l4.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 29257728, "nbytes": 3072 }, "stage.3.l4.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 16, "offset": 29260800, "nbytes": 1769472, "scales_offset": 31030272, "scales_nbytes": 221184 }, "stage.3.l4.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 16, "offset": 31251456, "nbytes": 589824, "scales_offset": 31841280, "scales_nbytes": 73728 }, "stage.3.l4.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 31915008, "nbytes": 3072 }, "stage.3.l4.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 31918080, "nbytes": 3072 }, "stage.3.l4.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 31921152, "nbytes": 3072 }, "stage.3.l4.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 16, "offset": 31924224, "nbytes": 2359296, "scales_offset": 34283520, "scales_nbytes": 294912 }, "stage.3.l4.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 16, "offset": 34578432, "nbytes": 2359296, "scales_offset": 36937728, "scales_nbytes": 294912 }, "stage.3.l4.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 37232640, "nbytes": 3072 }, "stage.3.l5.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 37235712, "nbytes": 3072 }, "stage.3.l5.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 37238784, "nbytes": 3072 }, "stage.3.l5.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 16, "offset": 37241856, "nbytes": 1769472, "scales_offset": 39011328, "scales_nbytes": 221184 }, "stage.3.l5.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 16, "offset": 39232512, "nbytes": 589824, "scales_offset": 39822336, "scales_nbytes": 73728 }, "stage.3.l5.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 39896064, "nbytes": 3072 }, "stage.3.l5.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 39899136, "nbytes": 3072 }, "stage.3.l5.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 39902208, "nbytes": 3072 }, "stage.3.l5.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 16, "offset": 39905280, "nbytes": 2359296, "scales_offset": 42264576, "scales_nbytes": 294912 }, "stage.3.l5.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 16, "offset": 42559488, "nbytes": 2359296, "scales_offset": 44918784, "scales_nbytes": 294912 }, "stage.3.l5.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 45213696, "nbytes": 3072 }, "stage.3.l6.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 45216768, "nbytes": 3072 }, "stage.3.l6.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 45219840, "nbytes": 3072 }, "stage.3.l6.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 16, "offset": 45222912, "nbytes": 1769472, "scales_offset": 46992384, "scales_nbytes": 221184 }, "stage.3.l6.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 16, "offset": 47213568, "nbytes": 589824, "scales_offset": 47803392, "scales_nbytes": 73728 }, "stage.3.l6.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 47877120, "nbytes": 3072 }, "stage.3.l6.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 47880192, "nbytes": 3072 }, "stage.3.l6.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 47883264, "nbytes": 3072 }, "stage.3.l6.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 16, "offset": 47886336, "nbytes": 2359296, "scales_offset": 50245632, "scales_nbytes": 294912 }, "stage.3.l6.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 16, "offset": 50540544, "nbytes": 2359296, "scales_offset": 52899840, "scales_nbytes": 294912 }, "stage.3.l6.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 53194752, "nbytes": 3072 }, "stage.3.l7.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 53197824, "nbytes": 3072 }, "stage.3.l7.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 53200896, "nbytes": 3072 }, "stage.3.l7.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 16, "offset": 53203968, "nbytes": 1769472, "scales_offset": 54973440, "scales_nbytes": 221184 }, "stage.3.l7.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 16, "offset": 55194624, "nbytes": 589824, "scales_offset": 55784448, "scales_nbytes": 73728 }, "stage.3.l7.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 55858176, "nbytes": 3072 }, "stage.3.l7.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 55861248, "nbytes": 3072 }, "stage.3.l7.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 55864320, "nbytes": 3072 }, "stage.3.l7.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 16, "offset": 55867392, "nbytes": 2359296, "scales_offset": 58226688, "scales_nbytes": 294912 }, "stage.3.l7.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 16, "offset": 58521600, "nbytes": 2359296, "scales_offset": 60880896, "scales_nbytes": 294912 }, "stage.3.l7.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 61175808, "nbytes": 3072 }, "stage.3.l8.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 61178880, "nbytes": 3072 }, "stage.3.l8.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 61181952, "nbytes": 3072 }, "stage.3.l8.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 16, "offset": 61185024, "nbytes": 1769472, "scales_offset": 62954496, "scales_nbytes": 221184 }, "stage.3.l8.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 16, "offset": 63175680, "nbytes": 589824, "scales_offset": 63765504, "scales_nbytes": 73728 }, "stage.3.l8.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 63839232, "nbytes": 3072 }, "stage.3.l8.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 63842304, "nbytes": 3072 }, "stage.3.l8.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 16, "offset": 63845376, "nbytes": 3072 }, "stage.3.l8.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 16, "offset": 63848448, "nbytes": 2359296, "scales_offset": 66207744, "scales_nbytes": 294912 }, "stage.3.l8.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 17, "offset": 0, "nbytes": 2359296, "scales_offset": 2359296, "scales_nbytes": 294912 }, "stage.3.l8.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 2654208, "nbytes": 3072 }, "stage.3.l9.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 2657280, "nbytes": 3072 }, "stage.3.l9.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 2660352, "nbytes": 3072 }, "stage.3.l9.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 17, "offset": 2663424, "nbytes": 1769472, "scales_offset": 4432896, "scales_nbytes": 221184 }, "stage.3.l9.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 17, "offset": 4654080, "nbytes": 589824, "scales_offset": 5243904, "scales_nbytes": 73728 }, "stage.3.l9.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 5317632, "nbytes": 3072 }, "stage.3.l9.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 5320704, "nbytes": 3072 }, "stage.3.l9.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 5323776, "nbytes": 3072 }, "stage.3.l9.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 17, "offset": 5326848, "nbytes": 2359296, "scales_offset": 7686144, "scales_nbytes": 294912 }, "stage.3.l9.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 17, "offset": 7981056, "nbytes": 2359296, "scales_offset": 10340352, "scales_nbytes": 294912 }, "stage.3.l9.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 10635264, "nbytes": 3072 }, "stage.3.l10.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 10638336, "nbytes": 3072 }, "stage.3.l10.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 10641408, "nbytes": 3072 }, "stage.3.l10.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 17, "offset": 10644480, "nbytes": 1769472, "scales_offset": 12413952, "scales_nbytes": 221184 }, "stage.3.l10.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 17, "offset": 12635136, "nbytes": 589824, "scales_offset": 13224960, "scales_nbytes": 73728 }, "stage.3.l10.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 13298688, "nbytes": 3072 }, "stage.3.l10.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 13301760, "nbytes": 3072 }, "stage.3.l10.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 13304832, "nbytes": 3072 }, "stage.3.l10.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 17, "offset": 13307904, "nbytes": 2359296, "scales_offset": 15667200, "scales_nbytes": 294912 }, "stage.3.l10.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 17, "offset": 15962112, "nbytes": 2359296, "scales_offset": 18321408, "scales_nbytes": 294912 }, "stage.3.l10.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 18616320, "nbytes": 3072 }, "stage.3.l11.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 18619392, "nbytes": 3072 }, "stage.3.l11.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 18622464, "nbytes": 3072 }, "stage.3.l11.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 17, "offset": 18625536, "nbytes": 1769472, "scales_offset": 20395008, "scales_nbytes": 221184 }, "stage.3.l11.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 17, "offset": 20616192, "nbytes": 589824, "scales_offset": 21206016, "scales_nbytes": 73728 }, "stage.3.l11.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 21279744, "nbytes": 3072 }, "stage.3.l11.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 21282816, "nbytes": 3072 }, "stage.3.l11.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 21285888, "nbytes": 3072 }, "stage.3.l11.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 17, "offset": 21288960, "nbytes": 2359296, "scales_offset": 23648256, "scales_nbytes": 294912 }, "stage.3.l11.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 17, "offset": 23943168, "nbytes": 2359296, "scales_offset": 26302464, "scales_nbytes": 294912 }, "stage.3.l11.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 26597376, "nbytes": 3072 }, "stage.4.in.weight": { "dtype": "q8", "shape": [ 384, 768 ], "shard": 17, "offset": 26600448, "nbytes": 294912, "scales_offset": 26895360, "scales_nbytes": 36864 }, "stage.4.l0.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 26932224, "nbytes": 3072 }, "stage.4.l0.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 26935296, "nbytes": 3072 }, "stage.4.l0.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 17, "offset": 26938368, "nbytes": 1769472, "scales_offset": 28707840, "scales_nbytes": 221184 }, "stage.4.l0.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 17, "offset": 28929024, "nbytes": 589824, "scales_offset": 29518848, "scales_nbytes": 73728 }, "stage.4.l0.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 29592576, "nbytes": 3072 }, "stage.4.l0.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 29595648, "nbytes": 3072 }, "stage.4.l0.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 29598720, "nbytes": 3072 }, "stage.4.l0.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 17, "offset": 29601792, "nbytes": 2359296, "scales_offset": 31961088, "scales_nbytes": 294912 }, "stage.4.l0.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 17, "offset": 32256000, "nbytes": 2359296, "scales_offset": 34615296, "scales_nbytes": 294912 }, "stage.4.l0.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 34910208, "nbytes": 3072 }, "stage.4.l1.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 34913280, "nbytes": 3072 }, "stage.4.l1.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 34916352, "nbytes": 3072 }, "stage.4.l1.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 17, "offset": 34919424, "nbytes": 1769472, "scales_offset": 36688896, "scales_nbytes": 221184 }, "stage.4.l1.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 17, "offset": 36910080, "nbytes": 589824, "scales_offset": 37499904, "scales_nbytes": 73728 }, "stage.4.l1.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 37573632, "nbytes": 3072 }, "stage.4.l1.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 37576704, "nbytes": 3072 }, "stage.4.l1.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 37579776, "nbytes": 3072 }, "stage.4.l1.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 17, "offset": 37582848, "nbytes": 2359296, "scales_offset": 39942144, "scales_nbytes": 294912 }, "stage.4.l1.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 17, "offset": 40237056, "nbytes": 2359296, "scales_offset": 42596352, "scales_nbytes": 294912 }, "stage.4.l1.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 42891264, "nbytes": 3072 }, "stage.4.l2.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 42894336, "nbytes": 3072 }, "stage.4.l2.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 42897408, "nbytes": 3072 }, "stage.4.l2.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 17, "offset": 42900480, "nbytes": 1769472, "scales_offset": 44669952, "scales_nbytes": 221184 }, "stage.4.l2.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 17, "offset": 44891136, "nbytes": 589824, "scales_offset": 45480960, "scales_nbytes": 73728 }, "stage.4.l2.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 45554688, "nbytes": 3072 }, "stage.4.l2.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 45557760, "nbytes": 3072 }, "stage.4.l2.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 45560832, "nbytes": 3072 }, "stage.4.l2.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 17, "offset": 45563904, "nbytes": 2359296, "scales_offset": 47923200, "scales_nbytes": 294912 }, "stage.4.l2.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 17, "offset": 48218112, "nbytes": 2359296, "scales_offset": 50577408, "scales_nbytes": 294912 }, "stage.4.l2.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 50872320, "nbytes": 3072 }, "stage.4.l3.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 50875392, "nbytes": 3072 }, "stage.4.l3.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 50878464, "nbytes": 3072 }, "stage.4.l3.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 17, "offset": 50881536, "nbytes": 1769472, "scales_offset": 52651008, "scales_nbytes": 221184 }, "stage.4.l3.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 17, "offset": 52872192, "nbytes": 589824, "scales_offset": 53462016, "scales_nbytes": 73728 }, "stage.4.l3.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 53535744, "nbytes": 3072 }, "stage.4.l3.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 53538816, "nbytes": 3072 }, "stage.4.l3.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 53541888, "nbytes": 3072 }, "stage.4.l3.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 17, "offset": 53544960, "nbytes": 2359296, "scales_offset": 55904256, "scales_nbytes": 294912 }, "stage.4.l3.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 17, "offset": 56199168, "nbytes": 2359296, "scales_offset": 58558464, "scales_nbytes": 294912 }, "stage.4.l3.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 58853376, "nbytes": 3072 }, "stage.4.l4.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 58856448, "nbytes": 3072 }, "stage.4.l4.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 58859520, "nbytes": 3072 }, "stage.4.l4.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 17, "offset": 58862592, "nbytes": 1769472, "scales_offset": 60632064, "scales_nbytes": 221184 }, "stage.4.l4.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 17, "offset": 60853248, "nbytes": 589824, "scales_offset": 61443072, "scales_nbytes": 73728 }, "stage.4.l4.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 61516800, "nbytes": 3072 }, "stage.4.l4.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 61519872, "nbytes": 3072 }, "stage.4.l4.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 61522944, "nbytes": 3072 }, "stage.4.l4.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 17, "offset": 61526016, "nbytes": 2359296, "scales_offset": 63885312, "scales_nbytes": 294912 }, "stage.4.l4.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 17, "offset": 64180224, "nbytes": 2359296, "scales_offset": 66539520, "scales_nbytes": 294912 }, "stage.4.l4.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 66834432, "nbytes": 3072 }, "stage.4.l5.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 66837504, "nbytes": 3072 }, "stage.4.l5.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 17, "offset": 66840576, "nbytes": 3072 }, "stage.4.l5.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 18, "offset": 0, "nbytes": 1769472, "scales_offset": 1769472, "scales_nbytes": 221184 }, "stage.4.l5.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 18, "offset": 1990656, "nbytes": 589824, "scales_offset": 2580480, "scales_nbytes": 73728 }, "stage.4.l5.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 2654208, "nbytes": 3072 }, "stage.4.l5.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 2657280, "nbytes": 3072 }, "stage.4.l5.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 2660352, "nbytes": 3072 }, "stage.4.l5.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 18, "offset": 2663424, "nbytes": 2359296, "scales_offset": 5022720, "scales_nbytes": 294912 }, "stage.4.l5.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 18, "offset": 5317632, "nbytes": 2359296, "scales_offset": 7676928, "scales_nbytes": 294912 }, "stage.4.l5.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 7971840, "nbytes": 3072 }, "stage.4.l6.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 7974912, "nbytes": 3072 }, "stage.4.l6.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 7977984, "nbytes": 3072 }, "stage.4.l6.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 18, "offset": 7981056, "nbytes": 1769472, "scales_offset": 9750528, "scales_nbytes": 221184 }, "stage.4.l6.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 18, "offset": 9971712, "nbytes": 589824, "scales_offset": 10561536, "scales_nbytes": 73728 }, "stage.4.l6.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 10635264, "nbytes": 3072 }, "stage.4.l6.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 10638336, "nbytes": 3072 }, "stage.4.l6.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 10641408, "nbytes": 3072 }, "stage.4.l6.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 18, "offset": 10644480, "nbytes": 2359296, "scales_offset": 13003776, "scales_nbytes": 294912 }, "stage.4.l6.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 18, "offset": 13298688, "nbytes": 2359296, "scales_offset": 15657984, "scales_nbytes": 294912 }, "stage.4.l6.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 15952896, "nbytes": 3072 }, "stage.4.l7.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 15955968, "nbytes": 3072 }, "stage.4.l7.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 15959040, "nbytes": 3072 }, "stage.4.l7.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 18, "offset": 15962112, "nbytes": 1769472, "scales_offset": 17731584, "scales_nbytes": 221184 }, "stage.4.l7.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 18, "offset": 17952768, "nbytes": 589824, "scales_offset": 18542592, "scales_nbytes": 73728 }, "stage.4.l7.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 18616320, "nbytes": 3072 }, "stage.4.l7.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 18619392, "nbytes": 3072 }, "stage.4.l7.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 18622464, "nbytes": 3072 }, "stage.4.l7.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 18, "offset": 18625536, "nbytes": 2359296, "scales_offset": 20984832, "scales_nbytes": 294912 }, "stage.4.l7.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 18, "offset": 21279744, "nbytes": 2359296, "scales_offset": 23639040, "scales_nbytes": 294912 }, "stage.4.l7.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 23933952, "nbytes": 3072 }, "stage.4.l8.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 23937024, "nbytes": 3072 }, "stage.4.l8.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 23940096, "nbytes": 3072 }, "stage.4.l8.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 18, "offset": 23943168, "nbytes": 1769472, "scales_offset": 25712640, "scales_nbytes": 221184 }, "stage.4.l8.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 18, "offset": 25933824, "nbytes": 589824, "scales_offset": 26523648, "scales_nbytes": 73728 }, "stage.4.l8.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 26597376, "nbytes": 3072 }, "stage.4.l8.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 26600448, "nbytes": 3072 }, "stage.4.l8.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 26603520, "nbytes": 3072 }, "stage.4.l8.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 18, "offset": 26606592, "nbytes": 2359296, "scales_offset": 28965888, "scales_nbytes": 294912 }, "stage.4.l8.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 18, "offset": 29260800, "nbytes": 2359296, "scales_offset": 31620096, "scales_nbytes": 294912 }, "stage.4.l8.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 31915008, "nbytes": 3072 }, "stage.4.l9.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 31918080, "nbytes": 3072 }, "stage.4.l9.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 31921152, "nbytes": 3072 }, "stage.4.l9.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 18, "offset": 31924224, "nbytes": 1769472, "scales_offset": 33693696, "scales_nbytes": 221184 }, "stage.4.l9.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 18, "offset": 33914880, "nbytes": 589824, "scales_offset": 34504704, "scales_nbytes": 73728 }, "stage.4.l9.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 34578432, "nbytes": 3072 }, "stage.4.l9.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 34581504, "nbytes": 3072 }, "stage.4.l9.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 34584576, "nbytes": 3072 }, "stage.4.l9.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 18, "offset": 34587648, "nbytes": 2359296, "scales_offset": 36946944, "scales_nbytes": 294912 }, "stage.4.l9.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 18, "offset": 37241856, "nbytes": 2359296, "scales_offset": 39601152, "scales_nbytes": 294912 }, "stage.4.l9.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 39896064, "nbytes": 3072 }, "stage.4.l10.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 39899136, "nbytes": 3072 }, "stage.4.l10.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 39902208, "nbytes": 3072 }, "stage.4.l10.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 18, "offset": 39905280, "nbytes": 1769472, "scales_offset": 41674752, "scales_nbytes": 221184 }, "stage.4.l10.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 18, "offset": 41895936, "nbytes": 589824, "scales_offset": 42485760, "scales_nbytes": 73728 }, "stage.4.l10.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 42559488, "nbytes": 3072 }, "stage.4.l10.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 42562560, "nbytes": 3072 }, "stage.4.l10.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 42565632, "nbytes": 3072 }, "stage.4.l10.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 18, "offset": 42568704, "nbytes": 2359296, "scales_offset": 44928000, "scales_nbytes": 294912 }, "stage.4.l10.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 18, "offset": 45222912, "nbytes": 2359296, "scales_offset": 47582208, "scales_nbytes": 294912 }, "stage.4.l10.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 47877120, "nbytes": 3072 }, "stage.4.l11.norm1.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 47880192, "nbytes": 3072 }, "stage.4.l11.norm1.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 47883264, "nbytes": 3072 }, "stage.4.l11.qkv.weight": { "dtype": "q8", "shape": [ 768, 2304 ], "shard": 18, "offset": 47886336, "nbytes": 1769472, "scales_offset": 49655808, "scales_nbytes": 221184 }, "stage.4.l11.wo.weight": { "dtype": "q8", "shape": [ 768, 768 ], "shard": 18, "offset": 49876992, "nbytes": 589824, "scales_offset": 50466816, "scales_nbytes": 73728 }, "stage.4.l11.attn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 50540544, "nbytes": 3072 }, "stage.4.l11.norm2.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 50543616, "nbytes": 3072 }, "stage.4.l11.norm2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 50546688, "nbytes": 3072 }, "stage.4.l11.ffn1.weight": { "dtype": "q8", "shape": [ 768, 3072 ], "shard": 18, "offset": 50549760, "nbytes": 2359296, "scales_offset": 52909056, "scales_nbytes": 294912 }, "stage.4.l11.ffn2.weight": { "dtype": "q8", "shape": [ 3072, 768 ], "shard": 18, "offset": 53203968, "nbytes": 2359296, "scales_offset": 55563264, "scales_nbytes": 294912 }, "stage.4.l11.ffn_scale": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 55858176, "nbytes": 3072 }, "head.0.dw.weight": { "dtype": "f32", "shape": [ 768, 7 ], "shard": 18, "offset": 55861248, "nbytes": 21504 }, "head.0.dw.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 55882752, "nbytes": 3072 }, "head.0.norm.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 55885824, "nbytes": 3072 }, "head.0.norm.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 55888896, "nbytes": 3072 }, "head.0.pw1.weight": { "dtype": "q8", "shape": [ 768, 2048 ], "shard": 18, "offset": 55891968, "nbytes": 1572864, "scales_offset": 57464832, "scales_nbytes": 196608 }, "head.0.pw1.bias": { "dtype": "f32", "shape": [ 2048 ], "shard": 18, "offset": 57661440, "nbytes": 8192 }, "head.0.pw2.weight": { "dtype": "q8", "shape": [ 2048, 768 ], "shard": 18, "offset": 57669632, "nbytes": 1572864, "scales_offset": 59242496, "scales_nbytes": 196608 }, "head.0.pw2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 59439104, "nbytes": 3072 }, "head.1.dw.weight": { "dtype": "f32", "shape": [ 768, 7 ], "shard": 18, "offset": 59442176, "nbytes": 21504 }, "head.1.dw.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 59463680, "nbytes": 3072 }, "head.1.norm.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 59466752, "nbytes": 3072 }, "head.1.norm.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 59469824, "nbytes": 3072 }, "head.1.pw1.weight": { "dtype": "q8", "shape": [ 768, 2048 ], "shard": 18, "offset": 59472896, "nbytes": 1572864, "scales_offset": 61045760, "scales_nbytes": 196608 }, "head.1.pw1.bias": { "dtype": "f32", "shape": [ 2048 ], "shard": 18, "offset": 61242368, "nbytes": 8192 }, "head.1.pw2.weight": { "dtype": "q8", "shape": [ 2048, 768 ], "shard": 18, "offset": 61250560, "nbytes": 1572864, "scales_offset": 62823424, "scales_nbytes": 196608 }, "head.1.pw2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 63020032, "nbytes": 3072 }, "head.2.dw.weight": { "dtype": "f32", "shape": [ 768, 7 ], "shard": 18, "offset": 63023104, "nbytes": 21504 }, "head.2.dw.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 63044608, "nbytes": 3072 }, "head.2.norm.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 63047680, "nbytes": 3072 }, "head.2.norm.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 63050752, "nbytes": 3072 }, "head.2.pw1.weight": { "dtype": "q8", "shape": [ 768, 2048 ], "shard": 18, "offset": 63053824, "nbytes": 1572864, "scales_offset": 64626688, "scales_nbytes": 196608 }, "head.2.pw1.bias": { "dtype": "f32", "shape": [ 2048 ], "shard": 18, "offset": 64823296, "nbytes": 8192 }, "head.2.pw2.weight": { "dtype": "q8", "shape": [ 2048, 768 ], "shard": 18, "offset": 64831488, "nbytes": 1572864, "scales_offset": 66404352, "scales_nbytes": 196608 }, "head.2.pw2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 66600960, "nbytes": 3072 }, "head.3.dw.weight": { "dtype": "f32", "shape": [ 768, 7 ], "shard": 18, "offset": 66604032, "nbytes": 21504 }, "head.3.dw.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 66625536, "nbytes": 3072 }, "head.3.norm.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 66628608, "nbytes": 3072 }, "head.3.norm.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 18, "offset": 66631680, "nbytes": 3072 }, "head.3.pw1.weight": { "dtype": "q8", "shape": [ 768, 2048 ], "shard": 19, "offset": 0, "nbytes": 1572864, "scales_offset": 1572864, "scales_nbytes": 196608 }, "head.3.pw1.bias": { "dtype": "f32", "shape": [ 2048 ], "shard": 19, "offset": 1769472, "nbytes": 8192 }, "head.3.pw2.weight": { "dtype": "q8", "shape": [ 2048, 768 ], "shard": 19, "offset": 1777664, "nbytes": 1572864, "scales_offset": 3350528, "scales_nbytes": 196608 }, "head.3.pw2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 19, "offset": 3547136, "nbytes": 3072 }, "head.4.dw.weight": { "dtype": "f32", "shape": [ 768, 7 ], "shard": 19, "offset": 3550208, "nbytes": 21504 }, "head.4.dw.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 19, "offset": 3571712, "nbytes": 3072 }, "head.4.norm.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 19, "offset": 3574784, "nbytes": 3072 }, "head.4.norm.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 19, "offset": 3577856, "nbytes": 3072 }, "head.4.pw1.weight": { "dtype": "q8", "shape": [ 768, 2048 ], "shard": 19, "offset": 3580928, "nbytes": 1572864, "scales_offset": 5153792, "scales_nbytes": 196608 }, "head.4.pw1.bias": { "dtype": "f32", "shape": [ 2048 ], "shard": 19, "offset": 5350400, "nbytes": 8192 }, "head.4.pw2.weight": { "dtype": "q8", "shape": [ 2048, 768 ], "shard": 19, "offset": 5358592, "nbytes": 1572864, "scales_offset": 6931456, "scales_nbytes": 196608 }, "head.4.pw2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 19, "offset": 7128064, "nbytes": 3072 }, "head.5.dw.weight": { "dtype": "f32", "shape": [ 768, 7 ], "shard": 19, "offset": 7131136, "nbytes": 21504 }, "head.5.dw.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 19, "offset": 7152640, "nbytes": 3072 }, "head.5.norm.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 19, "offset": 7155712, "nbytes": 3072 }, "head.5.norm.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 19, "offset": 7158784, "nbytes": 3072 }, "head.5.pw1.weight": { "dtype": "q8", "shape": [ 768, 2048 ], "shard": 19, "offset": 7161856, "nbytes": 1572864, "scales_offset": 8734720, "scales_nbytes": 196608 }, "head.5.pw1.bias": { "dtype": "f32", "shape": [ 2048 ], "shard": 19, "offset": 8931328, "nbytes": 8192 }, "head.5.pw2.weight": { "dtype": "q8", "shape": [ 2048, 768 ], "shard": 19, "offset": 8939520, "nbytes": 1572864, "scales_offset": 10512384, "scales_nbytes": 196608 }, "head.5.pw2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 19, "offset": 10708992, "nbytes": 3072 }, "head.6.dw.weight": { "dtype": "f32", "shape": [ 768, 7 ], "shard": 19, "offset": 10712064, "nbytes": 21504 }, "head.6.dw.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 19, "offset": 10733568, "nbytes": 3072 }, "head.6.norm.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 19, "offset": 10736640, "nbytes": 3072 }, "head.6.norm.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 19, "offset": 10739712, "nbytes": 3072 }, "head.6.pw1.weight": { "dtype": "q8", "shape": [ 768, 2048 ], "shard": 19, "offset": 10742784, "nbytes": 1572864, "scales_offset": 12315648, "scales_nbytes": 196608 }, "head.6.pw1.bias": { "dtype": "f32", "shape": [ 2048 ], "shard": 19, "offset": 12512256, "nbytes": 8192 }, "head.6.pw2.weight": { "dtype": "q8", "shape": [ 2048, 768 ], "shard": 19, "offset": 12520448, "nbytes": 1572864, "scales_offset": 14093312, "scales_nbytes": 196608 }, "head.6.pw2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 19, "offset": 14289920, "nbytes": 3072 }, "head.7.dw.weight": { "dtype": "f32", "shape": [ 768, 7 ], "shard": 19, "offset": 14292992, "nbytes": 21504 }, "head.7.dw.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 19, "offset": 14314496, "nbytes": 3072 }, "head.7.norm.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 19, "offset": 14317568, "nbytes": 3072 }, "head.7.norm.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 19, "offset": 14320640, "nbytes": 3072 }, "head.7.pw1.weight": { "dtype": "q8", "shape": [ 768, 2048 ], "shard": 19, "offset": 14323712, "nbytes": 1572864, "scales_offset": 15896576, "scales_nbytes": 196608 }, "head.7.pw1.bias": { "dtype": "f32", "shape": [ 2048 ], "shard": 19, "offset": 16093184, "nbytes": 8192 }, "head.7.pw2.weight": { "dtype": "q8", "shape": [ 2048, 768 ], "shard": 19, "offset": 16101376, "nbytes": 1572864, "scales_offset": 17674240, "scales_nbytes": 196608 }, "head.7.pw2.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 19, "offset": 17870848, "nbytes": 3072 }, "head.norm.weight": { "dtype": "f32", "shape": [ 768 ], "shard": 19, "offset": 17873920, "nbytes": 3072 }, "head.norm.bias": { "dtype": "f32", "shape": [ 768 ], "shard": 19, "offset": 17876992, "nbytes": 3072 }, "head.out.0.weight": { "dtype": "q8", "shape": [ 768, 738 ], "shard": 19, "offset": 17880064, "nbytes": 566784, "scales_offset": 18446848, "scales_nbytes": 70848 }, "head.out.0.bias": { "dtype": "f32", "shape": [ 738 ], "shard": 19, "offset": 18517760, "nbytes": 2952 }, "head.out.1.weight": { "dtype": "q8", "shape": [ 768, 2210 ], "shard": 19, "offset": 18520832, "nbytes": 1697280, "scales_offset": 20218112, "scales_nbytes": 212160 }, "head.out.1.bias": { "dtype": "f32", "shape": [ 2210 ], "shard": 19, "offset": 20430336, "nbytes": 8840 }, "head.mixer.weight": { "dtype": "f32", "shape": [ 127, 2 ], "shard": 19, "offset": 20439296, "nbytes": 1016, "host": true }, "head.window.0": { "dtype": "f32", "shape": [ 736 ], "shard": 19, "offset": 20440320, "nbytes": 2944, "host": true }, "head.window.1": { "dtype": "f32", "shape": [ 2208 ], "shard": 19, "offset": 20443392, "nbytes": 8832, "host": true }, "resample.kernel": { "dtype": "f32", "shape": [ 770, 735 ], "shard": 19, "offset": 20452352, "nbytes": 2263800, "host": true } } }