# coding=utf-8
# Copyright 2026 the Maxwell team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Maxwell MoE model configuration with mHC, QK LayerNorm, expert bias support."""

from transformers.configuration_utils import PretrainedConfig
from transformers.modeling_rope_utils import rope_config_validation
from transformers.utils import logging

logger = logging.get_logger(__name__)


class MaxwellMoeConfig(PretrainedConfig):
    model_type = "maxwell_moe"
    keys_to_ignore_at_inference = ["past_key_values"]

    def __init__(
        self,
        vocab_size=200704,
        hidden_size=2048,
        intermediate_size=6144,
        num_hidden_layers=45,
        num_attention_heads=32,
        num_key_value_heads=4,
        attention_head_size=128,
        hidden_act="silu",
        max_position_embeddings=524288,
        initializer_range=0.02,
        rms_norm_eps=1e-6,
        use_cache=True,
        tie_word_embeddings=False,
        rope_theta=20000000,
        rope_scaling=None,
        attention_variant="block_sparse_gqa",
        block_sparse_gqa_indexer_n_heads=4,
        block_sparse_gqa_indexer_head_dim=128,
        block_sparse_gqa_block_size=128,
        # Remote blocks are selected independently per GQA group using the
        # maximum token score within each block (fixed architecture semantics).
        # Prefix/local block counts are derived from window_size / block_size.
        block_sparse_gqa_prefix_window_size=1024,
        block_sparse_gqa_local_window_size=1024,
        block_sparse_gqa_num_remote_blocks=16,
        block_sparse_gqa_layer_freq=None,
        # Inference execution tuning: query tokens processed per chunk by
        # indexer block selection and eager sparse attention. Must be positive.
        # Smaller chunks reduce chunk-local temporary memory; larger chunks
        # may reduce loop overhead. This is not the sparse block size or budget
        # and does not change the intended attention pattern.
        block_sparse_gqa_query_chunk_size=64,
        attention_dropout=0.0,
        moe_intermediate_size=512,
        shared_expert_intermediate_size=1024,
        num_experts_per_tok=12,
        num_experts=256,
        output_router_logits=False,
        router_aux_loss_coef=0.0,
        mlp_only_layers=None,
        use_shared_expert_gate=False,
        # QK LayerNorm
        qk_layernorm=True,
        # mHC parameters
        enable_hyper_connections=True,
        num_residual_streams=4,
        mhc_sinkhorn_iterations=20,
        mhc_init_gating_factor=0.01,
        # Router
        moe_router_enable_expert_bias=True,
        router_score_function="sigmoid",
        # MTP
        mtp_num_layers=0,
        **kwargs,
    ):
        # Sliding-window attention (SWA) is not supported.
        if kwargs.pop("use_sliding_window", False):
            raise ValueError("Sliding-window attention is not supported.")
        kwargs.pop("sliding_window", None)
        kwargs.pop("max_window_layers", None)
        compression_fields = sorted(
            key
            for key in kwargs
            if key.startswith("block_sparse_gqa_compression_")
        )
        if compression_fields:
            raise ValueError(
                "This template is raw-token-only and rejects compression "
                f"fields: {compression_fields}"
            )
        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.intermediate_size = intermediate_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.num_key_value_heads = num_key_value_heads
        self.attention_head_size = attention_head_size
        self.max_position_embeddings = max_position_embeddings

        if attention_variant != "block_sparse_gqa":
            raise ValueError(
                "This template requires `attention_variant='block_sparse_gqa'`."
            )
        self.attention_variant = attention_variant

        self.block_sparse_gqa_indexer_n_heads = int(
            block_sparse_gqa_indexer_n_heads
        )
        self.block_sparse_gqa_indexer_head_dim = int(
            block_sparse_gqa_indexer_head_dim
        )
        self.block_sparse_gqa_block_size = int(block_sparse_gqa_block_size)
        # The indexer reuses the main attention's RoPE cos/sin tensors;
        # rope_theta is the single source of truth for their rotary base.
        self.block_sparse_gqa_prefix_window_size = int(
            block_sparse_gqa_prefix_window_size
        )
        self.block_sparse_gqa_local_window_size = int(
            block_sparse_gqa_local_window_size
        )
        self.block_sparse_gqa_num_remote_blocks = int(
            block_sparse_gqa_num_remote_blocks
        )
        if block_sparse_gqa_layer_freq is None:
            block_sparse_gqa_layer_freq = [1] * num_hidden_layers
        self.block_sparse_gqa_layer_freq = list(
            block_sparse_gqa_layer_freq
        )
        self.block_sparse_gqa_query_chunk_size = int(
            block_sparse_gqa_query_chunk_size
        )

        if self.block_sparse_gqa_indexer_n_heads != self.num_key_value_heads:
            raise ValueError(
                "block_sparse_gqa requires one index query head per GQA group"
            )
        if self.block_sparse_gqa_indexer_head_dim != self.attention_head_size:
            raise ValueError(
                "block_sparse_gqa indexer_head_dim must equal attention_head_size"
            )
        if self.block_sparse_gqa_block_size <= 0:
            raise ValueError(
                "block_sparse_gqa_block_size must be positive"
            )
        if self.block_sparse_gqa_query_chunk_size <= 0:
            raise ValueError(
                "block_sparse_gqa_query_chunk_size must be positive"
            )
        if self.block_sparse_gqa_prefix_window_size <= 0:
            raise ValueError(
                "block_sparse_gqa_prefix_window_size must be positive"
            )
        if self.block_sparse_gqa_local_window_size <= 0:
            raise ValueError(
                "block_sparse_gqa_local_window_size must be positive"
            )
        if (
            self.block_sparse_gqa_prefix_window_size
            % self.block_sparse_gqa_block_size
        ):
            raise ValueError(
                "block_sparse_gqa_prefix_window_size must be block aligned"
            )
        if (
            self.block_sparse_gqa_local_window_size
            % self.block_sparse_gqa_block_size
        ):
            raise ValueError(
                "block_sparse_gqa_local_window_size must be block aligned"
            )
        if self.block_sparse_gqa_num_remote_blocks <= 0:
            raise ValueError(
                "block_sparse_gqa_num_remote_blocks must be positive"
            )
        expected_layer_freq = [1] * self.num_hidden_layers
        if self.block_sparse_gqa_layer_freq != expected_layer_freq:
            raise ValueError(
                "This template requires every decoder layer to use "
                "Block Sparse GQA"
            )
        if attention_dropout != 0.0:
            raise ValueError(
                "block_sparse_gqa requires attention_dropout=0"
            )
        if self.num_attention_heads % self.num_key_value_heads:
            raise ValueError(
                "num_attention_heads must be divisible by num_key_value_heads"
            )

        self.hidden_act = hidden_act
        self.initializer_range = initializer_range
        self.rms_norm_eps = rms_norm_eps
        self.use_cache = use_cache
        self.rope_theta = rope_theta
        self.rope_scaling = rope_scaling
        self.attention_dropout = attention_dropout
        if self.rope_scaling is not None and "type" in self.rope_scaling:
            self.rope_scaling["rope_type"] = self.rope_scaling["type"]
        rope_config_validation(self)

        self.moe_intermediate_size = moe_intermediate_size
        self.shared_expert_intermediate_size = shared_expert_intermediate_size
        self.num_experts_per_tok = num_experts_per_tok
        self.num_experts = num_experts
        self.output_router_logits = output_router_logits
        self.router_aux_loss_coef = router_aux_loss_coef
        self.mlp_only_layers = [] if mlp_only_layers is None else mlp_only_layers
        self.use_shared_expert_gate = use_shared_expert_gate

        self.qk_layernorm = qk_layernorm

        self.enable_hyper_connections = enable_hyper_connections
        self.num_residual_streams = num_residual_streams
        self.mhc_sinkhorn_iterations = mhc_sinkhorn_iterations
        self.mhc_init_gating_factor = mhc_init_gating_factor

        self.moe_router_enable_expert_bias = moe_router_enable_expert_bias
        self.router_score_function = router_score_function

        self.mtp_num_layers = mtp_num_layers

        super().__init__(
            tie_word_embeddings=tie_word_embeddings,
            **kwargs,
        )
