{ "model": { "architecture": "s1_semantic_rvq_moss_v2_hybrid", "sample_rate": 44100, "internal_sample_rate": 44160, "channels": 1, "channel_interleave": false, "frame_rate": 15, "internal_hop_length": 2944, "external_samples_per_frame": 2940, "encoder_stages": [ { "type": "S1CausalWNConv1d", "in_channels": 1, "out_channels": 64, "kernel_size": 7, "stride": 1 }, { "type": "S1EncoderBlock", "in_channels": 64, "out_channels": 128, "stride": 2, "kernel_size": 4, "residual_dilations": [ 1, 3, 9 ], "residual_kernel_size": 7, "activation": "Snake1d", "transformer_layers": 0 }, { "type": "S1EncoderBlock", "in_channels": 128, "out_channels": 256, "stride": 4, "kernel_size": 8, "residual_dilations": [ 1, 3, 9 ], "residual_kernel_size": 7, "activation": "Snake1d", "transformer_layers": 0 }, { "type": "PatchedPretransform", "patch_size": 23, "direction": "down" }, { "type": "MossProjectedTransformer", "stage": 2, "input_dimension": 5888, "output_dimension": 384, "d_model": 768, "num_heads": 12, "num_layers": 12, "dim_feedforward": 3072, "causal": true, "norm": "layer_norm", "positional_embedding": "rope", "max_period": 10000, "gating": "none", "activation": "gelu", "layer_scale": 0.01, "attention_dropout": 0.1, "ffn_dropout": 0.1, "frame_rate": 240, "context_duration": 2, "context_frames": 480 }, { "type": "PatchedPretransform", "patch_size": 2, "direction": "down" }, { "type": "MossProjectedTransformer", "stage": 3, "input_dimension": 768, "output_dimension": 384, "d_model": 768, "num_heads": 12, "num_layers": 12, "dim_feedforward": 3072, "causal": true, "norm": "layer_norm", "positional_embedding": "rope", "max_period": 10000, "gating": "none", "activation": "gelu", "layer_scale": 0.01, "attention_dropout": 0.1, "ffn_dropout": 0.1, "frame_rate": 120, "context_duration": 4, "context_frames": 480 }, { "type": "PatchedPretransform", "patch_size": 2, "direction": "down" }, { "type": "MossProjectedTransformer", "stage": 4, "input_dimension": 768, "output_dimension": 384, "d_model": 768, "num_heads": 12, "num_layers": 12, "dim_feedforward": 3072, "causal": true, "norm": "layer_norm", "positional_embedding": "rope", "max_period": 10000, "gating": "none", "activation": "gelu", "layer_scale": 0.01, "attention_dropout": 0.1, "ffn_dropout": 0.1, "frame_rate": 60, "context_duration": 8, "context_frames": 480 }, { "type": "PatchedPretransform", "patch_size": 2, "direction": "down" }, { "type": "MossProjectedTransformer", "stage": 5, "input_dimension": 768, "output_dimension": 640, "d_model": 768, "num_heads": 12, "num_layers": 12, "dim_feedforward": 3072, "causal": true, "norm": "layer_norm", "positional_embedding": "rope", "max_period": 10000, "gating": "none", "activation": "gelu", "layer_scale": 0.01, "attention_dropout": 0.1, "ffn_dropout": 0.1, "frame_rate": 30, "context_duration": 10, "context_frames": 300 }, { "type": "PatchedPretransform", "patch_size": 2, "direction": "down" }, { "type": "MossProjectedTransformer", "stage": 6, "input_dimension": 1280, "output_dimension": 1024, "d_model": 1280, "num_heads": 20, "num_layers": 32, "dim_feedforward": 5120, "causal": true, "norm": "layer_norm", "positional_embedding": "rope", "max_period": 10000, "gating": "none", "activation": "gelu", "layer_scale": 0.01, "attention_dropout": 0.1, "ffn_dropout": 0.1, "frame_rate": 15, "context_duration": 10, "context_frames": 150 } ], "decoder_stages": [ { "type": "MossProjectedTransformer", "stage": 6, "input_dimension": 1024, "output_dimension": 1280, "d_model": 1280, "num_heads": 20, "num_layers": 32, "dim_feedforward": 5120, "causal": true, "norm": "layer_norm", "positional_embedding": "rope", "max_period": 10000, "gating": "none", "activation": "gelu", "layer_scale": 0.01, "attention_dropout": 0.1, "ffn_dropout": 0.1, "frame_rate": 15, "context_duration": 10, "context_frames": 150 }, { "type": "PatchedPretransform", "patch_size": 2, "direction": "up" }, { "type": "MossProjectedTransformer", "stage": 5, "input_dimension": 640, "output_dimension": 768, "d_model": 768, "num_heads": 12, "num_layers": 12, "dim_feedforward": 3072, "causal": true, "norm": "layer_norm", "positional_embedding": "rope", "max_period": 10000, "gating": "none", "activation": "gelu", "layer_scale": 0.01, "attention_dropout": 0.1, "ffn_dropout": 0.1, "frame_rate": 30, "context_duration": 10, "context_frames": 300 }, { "type": "PatchedPretransform", "patch_size": 2, "direction": "up" }, { "type": "MossProjectedTransformer", "stage": 4, "input_dimension": 384, "output_dimension": 768, "d_model": 768, "num_heads": 12, "num_layers": 12, "dim_feedforward": 3072, "causal": true, "norm": "layer_norm", "positional_embedding": "rope", "max_period": 10000, "gating": "none", "activation": "gelu", "layer_scale": 0.01, "attention_dropout": 0.1, "ffn_dropout": 0.1, "frame_rate": 60, "context_duration": 8, "context_frames": 480 }, { "type": "PatchedPretransform", "patch_size": 2, "direction": "up" }, { "type": "MossProjectedTransformer", "stage": 3, "input_dimension": 384, "output_dimension": 768, "d_model": 768, "num_heads": 12, "num_layers": 12, "dim_feedforward": 3072, "causal": true, "norm": "layer_norm", "positional_embedding": "rope", "max_period": 10000, "gating": "none", "activation": "gelu", "layer_scale": 0.01, "attention_dropout": 0.1, "ffn_dropout": 0.1, "frame_rate": 120, "context_duration": 4, "context_frames": 480 }, { "type": "PatchedPretransform", "patch_size": 2, "direction": "up" }, { "type": "MossProjectedTransformer", "stage": 2, "input_dimension": 384, "output_dimension": 5888, "d_model": 768, "num_heads": 12, "num_layers": 12, "dim_feedforward": 3072, "causal": true, "norm": "layer_norm", "positional_embedding": "rope", "max_period": 10000, "gating": "none", "activation": "gelu", "layer_scale": 0.01, "attention_dropout": 0.1, "ffn_dropout": 0.1, "frame_rate": 240, "context_duration": 2, "context_frames": 480 }, { "type": "PatchedPretransform", "patch_size": 23, "direction": "up" }, { "type": "S1DecoderBlock", "in_channels": 256, "out_channels": 128, "stride": 4, "kernel_size": 8, "residual_dilations": [ 1, 3, 9 ], "residual_kernel_size": 7, "activation": "Snake1d", "transformer_layers": 0 }, { "type": "S1DecoderBlock", "in_channels": 128, "out_channels": 64, "stride": 2, "kernel_size": 4, "residual_dilations": [ 1, 3, 9 ], "residual_kernel_size": 7, "activation": "Snake1d", "transformer_layers": 0 }, { "type": "S1WaveformHead", "in_channels": 64, "out_channels": 1, "kernel_size": 7, "activation_before": "Snake1d", "activation_after": "tanh" } ], "bottleneck": { "type": "dual_encoding_semantic_residual_rvq", "input_dim": 1024, "semantic_codebooks": 1, "residual_codebooks": 9, "semantic_codebook_size": 4096, "residual_codebook_size": 4096, "codebook_dim": 8, "quantizer_dropout": 0.5, "semantic_quantizer_dropout": 0.0, "downsample_factors": [], "upsample_factors": [], "pre_module": "identity", "post_module": "identity", "lookup_and_vq_loss_dtype": "float32", "semantic_head": { "type": "ConvNeXtDecoder", "width": 768, "hidden": 2048, "depth": 4, "out_channels": 1024, "causal": true }, "distillation_input": "decoded_ssl_semantic_codebook" }, "boundary_resampler": { "input_rate": 44100, "codec_rate": 44160, "output_rate": 44100, "encoder_ratio": [ 736, 735 ], "decoder_ratio": [ 735, 736 ], "method": "sinc_interp_hann", "lowpass_filter_width": 16, "rolloff": 0.99, "streaming_policy": "finite_fir_lookahead_with_delay_compensated_timestamps", "training_policy": "time_aligned_full_crop_reference", "losses_and_teacher_use_external_rate": true }, "checkpoint_transformers": true, "attention_implementation": "pytorch_sdpa_chunked_causal_local_with_dropout", "residual_codebooks": 9, "semantic_codebook_size": 4096, "residual_codebook_size": 4096, "codebook_dim": 8, "quantizer_dropout": 0.5, "decoder_mirrors_encoder": true, "dual_encoding": { "width": 768, "hidden": 2048, "depth": 4, "causal": true, "window_ratio": 2.0, "strategy": "frozen_ssl_to_q1_decoded_semantic_residual", "normalization": "unchanged_baseline" } }, "head": { "depth": 8, "hidden": 2048, "hop": 184, "frame_rate": 240, "trainable_stages": [ 0, 1, 2, 3, 4, 5, 6, 7, 8 ], "parametrization": "per branch: log-magnitude + phase angle (Vocos), magnitude clipped at 1e2, periodic Hann, centered iSTFT; branches combined by a linear fp32 FIR mixer", "recompute_trainable_stages": true, "compile_trainable_stages": true, "type": "mr_istft", "n_ffts": [ 736, 2208 ], "fir_kernel": 127, "warm_blocks": 4 }, "init": { "generator_checkpoint": "hf://Zyphra/ZONOS3-Mini/autoencoder/generator.pt", "trainable_rename": { "head.out.": "head.outs.0." } } }