# Copyright 2025 The Qwen Team and The HuggingFace Inc. team. All rights reserved.
# Copyright 2026 Cloudflare, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software distributed
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.

"""Numerical operations for the Transformers Clairic backbone."""

from __future__ import annotations

from collections.abc import Callable
from types import MethodType
from typing import Any

import torch
from torch import nn
from torch.nn import functional as F
from torch.utils._python_dispatch import TorchDispatchMode
from transformers.cache_utils import Cache
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
from transformers.models.qwen3_5.modeling_qwen3_5 import (
    Qwen3_5Attention, Qwen3_5DecoderLayer, Qwen3_5ForConditionalGeneration,
    Qwen3_5GatedDeltaNet, Qwen3_5MLP, Qwen3_5RMSNorm,
    Qwen3_5TextRotaryEmbedding, apply_mask_to_padding_states,
    eager_attention_forward, torch_causal_conv1d_update,
)

from .attention_metadata import DecodeMetadata
from .recurrent_metadata import scope as recurrent_metadata_scope


def _matmul(a: torch.Tensor, b: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor:
    if a.ndim != 2 or b.ndim != 2 or a.shape[1] != b.shape[0] or a.dtype != b.dtype:
        raise ValueError("Expected aligned matrices with equal dtype")
    if (a.dtype == torch.bfloat16 and b.dtype == torch.bfloat16
            and a.is_contiguous() and b.transpose(0, 1).is_contiguous()
            and b.shape[1] >= 16):
        from deep_gemm import bf16_gemm_nn
        output = torch.empty((a.shape[0], b.shape[1]), device=a.device, dtype=a.dtype)
        bf16_gemm_nn(a, b, output)
        if bias is not None:
            output += bias
        return output
    from .kernels import _matmul_persistent_triton
    return _matmul_persistent_triton(a, b, bias)


def _first_tensor(values: Any) -> torch.Tensor | None:
    if isinstance(values, torch.Tensor):
        return values
    if isinstance(values, (list, tuple)):
        return next((found for value in values if (found := _first_tensor(value)) is not None), None)
    return None


class ReadoutMath(TorchDispatchMode):
    """Apply CUDA readout operations within the current worker call."""

    def __torch_dispatch__(self: ReadoutMath, function: Any, types: Any, args: tuple[Any, ...] = (),
                           kwargs: dict[str, Any] | None = None) -> Any:
        kwargs = {} if kwargs is None else kwargs
        first = _first_tensor(args)
        eligible = first is not None and first.is_cuda and first.dtype in (torch.bfloat16, torch.float32)
        if eligible and function in (torch.ops.aten.linear.default, torch.ops.aten.matmul.default,
                                      torch.ops.aten.einsum.default,
                                      torch.ops.aten.scaled_dot_product_attention.default,
                                      torch.ops.aten._scaled_dot_product_attention_math.default):
            with self:
                result = function.decompose(*args, **kwargs)
            if result is NotImplemented:
                raise RuntimeError(f"Missing matrix decomposition: {function}")
            return result
        if eligible and function is torch.ops.aten._native_multi_head_attention.default:
            with self:
                return function._op_dk(torch._C.DispatchKey.CUDA, *args, **kwargs)
        if eligible and function is torch.ops.aten.mm.default:
            return _matmul(args[0], args[1])
        if eligible and function is torch.ops.aten.addmm.default:
            if kwargs.get("alpha", 1) != 1 or kwargs.get("beta", 1) != 1:
                raise ValueError("Expected unit addmm alpha and beta")
            return _matmul(args[1], args[2], args[0])
        if eligible and function is torch.ops.aten.mm.dtype:
            return _matmul(args[0].contiguous(), args[1].contiguous()).to(kwargs["out_dtype"])
        if eligible and function is torch.ops.aten.bmm.default:
            from .kernels import bmm_batch_invariant
            return bmm_batch_invariant(args[0], args[1])
        if eligible and function is torch.ops.aten.mean.dim:
            from .kernels import mean_batch_invariant
            return mean_batch_invariant(args[0], args[1], args[2] if len(args) > 2 else False,
                                        dtype=kwargs.get("dtype"))
        return function(*args, **kwargs)


def _linear_forward(original: Callable[..., Any], lm_head: bool,
                    transpose_caches: list[dict[str, Any]] | None = None) -> Callable[..., torch.Tensor]:
    cache: dict[str, Any] = {}
    if lm_head and transpose_caches is not None:
        transpose_caches.append(cache)
    def forward(self: Any, input: torch.Tensor) -> torch.Tensor:
        if (not input.is_cuda or input.dtype not in (torch.bfloat16, torch.float32)
                or input.dtype != self.weight.dtype):
            cache.clear()
            return original(input)
        shape = input.shape[:-1]
        flat = input.reshape(-1, input.shape[-1])
        if lm_head:
            weight = self.weight
            version = None
            if not torch.is_grad_enabled() and not torch.compiler.is_compiling():
                try:
                    version = weight._version
                except RuntimeError:
                    pass
            if version is None:
                cache.clear()
                transposed = weight.T.contiguous()
            else:
                stream = torch.cuda.current_stream(weight.device).cuda_stream if weight.is_cuda else None
                key = (id(weight), version, weight.data_ptr(), weight.storage_offset(),
                       weight.device, weight.dtype, tuple(weight.shape), tuple(weight.stride()), stream)
                if cache.get("weight") is not weight or cache.get("key") != key:
                    cache.clear()
                    cache["transposed"] = weight.T.contiguous()
                    cache["key"] = key
                    cache["weight"] = weight
                transposed = cache["transposed"]
            output = _matmul(flat.contiguous(), transposed).float()
        else:
            output = _matmul(flat, self.weight.T, self.bias)
        return output.reshape(*shape, self.weight.shape[0])
    return forward


def _norm(self: Qwen3_5RMSNorm, x: torch.Tensor) -> torch.Tensor:
    if x.is_cuda and x.dtype in (torch.bfloat16, torch.float32):
        from .normalization import rms_norm
        return rms_norm(x, self.weight, self.eps)
    value = x.float()
    value = value * torch.rsqrt(value.square().mean(-1, keepdim=True) + self.eps)
    return (value * (1.0 + self.weight.float())).to(self.weight.dtype)


def _residual_sum(first: torch.Tensor, second: torch.Tensor,
                  dtype: torch.dtype) -> torch.Tensor:
    if (dtype == torch.bfloat16 and first.is_cuda and second.is_cuda
            and first.device == second.device and first.shape == second.shape
            and first.dtype in (torch.bfloat16, torch.float32)
            and second.dtype == torch.bfloat16
            and first.is_contiguous() and second.is_contiguous()
            and first.numel() <= 2**31 - 1
            and not torch.is_grad_enabled() and not torch.compiler.is_compiling()):
        from .kernels import residual_sum
        return residual_sum(first, second)
    return first.to(dtype).float() + second.float()



def _decoder(self: Qwen3_5DecoderLayer, hidden_states: torch.Tensor,
             position_embeddings: tuple[torch.Tensor, torch.Tensor],
             attention_mask: torch.Tensor | None = None,
             position_ids: torch.Tensor | None = None, past_key_values: Cache | None = None,
             **kwargs: Any) -> torch.Tensor:
    dtype = self.input_layernorm.weight.dtype
    residual = hidden_states.to(dtype)
    states = self.input_layernorm(hidden_states)
    if self.layer_type == "linear_attention":
        states = self.linear_attn(hidden_states=states, cache_params=past_key_values,
                                  attention_mask=attention_mask, **kwargs)
    else:
        states, _ = self.self_attn(hidden_states=states, attention_mask=attention_mask,
            position_ids=position_ids, past_key_values=past_key_values,
            position_embeddings=position_embeddings, **kwargs)
    merged = _residual_sum(residual, states, dtype)
    states = self.post_attention_layernorm(merged)
    return _residual_sum(merged, self.mlp(states), dtype)


def _mlp(self: Qwen3_5MLP, x: torch.Tensor) -> torch.Tensor:
    gate, up = self.gate_proj(x), self.up_proj(x)
    if (gate.is_cuda and gate.dtype == up.dtype == torch.bfloat16
            and gate.is_contiguous() and up.is_contiguous()):
        from .kernels import silu_mul
        return self.down_proj(silu_mul(gate, up))
    return self.down_proj((F.silu(gate.float()) * up.float()).to(gate.dtype))


def _rotate(q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor,
            sin: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
    dim = cos.shape[-1]
    half = dim // 2
    c, s = cos[:, None, :, :half], sin[:, None, :, :half]

    def apply(value: torch.Tensor) -> torch.Tensor:
        first, second = value[..., :half].float(), value[..., half:dim].float()
        rotated = torch.cat((first * c - second * s, second * c + first * s), -1)
        return torch.cat((rotated.to(value.dtype), value[..., dim:]), -1)

    return apply(q), apply(k)


def _dense(self: Qwen3_5Attention, hidden_states: torch.Tensor,
           position_embeddings: tuple[torch.Tensor, torch.Tensor],
           attention_mask: torch.Tensor | None, past_key_values: Cache | None = None,
           *, metadata: DecodeMetadata | None = None,
           **kwargs: Any) -> tuple[torch.Tensor, torch.Tensor | None]:
    input_shape = hidden_states.shape[:-1]
    hidden_shape = (*input_shape, -1, self.head_dim)
    q_gate, raw_key = self.q_proj(hidden_states), self.k_proj(hidden_states)
    if hidden_states.is_cuda:
        from .kernels import dense_qk
        query, key, gate = dense_qk(q_gate, raw_key, self.q_norm.weight,
            self.k_norm.weight, *position_embeddings, self.q_norm.eps)
    else:
        query, gate = torch.chunk(q_gate.view(*input_shape, -1, self.head_dim * 2), 2, dim=-1)
        gate = gate.reshape(*input_shape, -1)
        query = self.q_norm(query.view(hidden_shape)).transpose(1, 2)
        key = self.k_norm(raw_key.view(hidden_shape)).transpose(1, 2)
        query, key = _rotate(query, key, *position_embeddings)
    value = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
    if past_key_values is not None:
        key, value = past_key_values.update(key, value, self.layer_idx)
    interface = ALL_ATTENTION_FUNCTIONS.get_interface(
        self.config._attn_implementation, eager_attention_forward)
    if metadata is not None:
        output, weights = metadata.attend(interface, self, query, key, value, attention_mask,
            dropout=0.0 if not self.training else self.attention_dropout,
            scaling=self.scaling, **kwargs)
    else:
        output, weights = interface(self, query, key, value, attention_mask,
            dropout=0.0 if not self.training else self.attention_dropout,
            scaling=self.scaling, **kwargs)
    output = output.reshape(*input_shape, -1).contiguous()
    if output.is_cuda and output.dtype == torch.bfloat16:
        from .kernels import fused_sigmoid_mul
        output = fused_sigmoid_mul(output, gate, inplace=True)
    else:
        output = (output.float() * torch.sigmoid(gate.float())).to(output.dtype)
    return self.o_proj(output), weights


def _gdn(self: Qwen3_5GatedDeltaNet, hidden_states: torch.Tensor,
         cache_params: Cache | None = None, attention_mask: torch.Tensor | None = None,
         **kwargs: Any) -> torch.Tensor:
    hidden_states = apply_mask_to_padding_states(hidden_states, attention_mask)
    batch, length, _ = hidden_states.shape
    cached = cache_params is not None and cache_params.has_previous_state(self.layer_idx)
    if cached:
        conv_state = cache_params.layers[self.layer_idx].conv_states
        recurrent_state = cache_params.layers[self.layer_idx].recurrent_states
    mixed = self.in_proj_qkv(hidden_states).transpose(1, 2)
    z = self.in_proj_z(hidden_states).reshape(batch, length, -1, self.head_v_dim)
    b, a = self.in_proj_b(hidden_states), self.in_proj_a(hidden_states)
    if cached and length == 1:
        mixed = self.causal_conv1d_update(mixed, conv_state, self.conv1d.weight.squeeze(1),
                                         self.conv1d.bias, self.activation)
    else:
        if cached:
            mixed = torch.cat((conv_state, mixed), -1)
        if cache_params is not None:
            state = F.pad(mixed, (self.conv_kernel_size - mixed.shape[-1], 0))
            cache_params.update_conv_state(state, self.layer_idx)
        if self.causal_conv1d_fn is not None:
            mixed = self.causal_conv1d_fn(x=mixed, weight=self.conv1d.weight.squeeze(1),
                bias=self.conv1d.bias, activation=self.activation, seq_idx=kwargs.get("seq_idx"))
        else:
            mixed = F.silu(self.conv1d(mixed)[:, :, :mixed.shape[-1]])
        if cached:
            mixed = mixed[:, :, -length:]
    from .gdn.adapter import decode, prefill, store_state
    if cached and length == 1:
        output, state = decode(mixed.transpose(1, 2), a, b, self.A_log, self.dt_bias,
                               recurrent_state)
    else:
        output, state = prefill(mixed.transpose(1, 2), a, b, self.A_log, self.dt_bias,
            self.num_k_heads, self.num_v_heads, self.head_k_dim, self.head_v_dim,
            recurrent_state if cached else None, attention_mask)
    if cache_params is not None:
        store_state(cache_params, self.layer_idx, state)
    output = self.norm(output.reshape(-1, self.head_v_dim), z.reshape(-1, self.head_v_dim))
    return self.out_proj(output.reshape(batch, length, -1))


def _gated_norm(self: Any, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
    epsilon = getattr(self, "eps", getattr(self, "variance_epsilon", None))
    if (x.ndim != 2 or z.shape != x.shape or self.weight.shape != (128,)
            or x.shape[-1] != 128 or x.shape[0] < 1 or epsilon != 1e-6
            or x.dtype != z.dtype or self.weight.dtype != x.dtype
            or any(v.device != x.device or v.stride(-1) != 1 for v in (x, z, self.weight))):
        raise ValueError("Unsupported gated normalization inputs")
    from .kernels import gated_norm
    return gated_norm(x, z, self.weight, epsilon)


def _nonoverlapping(value: torch.Tensor) -> None:
    span = 1
    for stride, size in sorted((s, n) for s, n in zip(value.stride(), value.shape, strict=True) if n > 1):
        if stride < span:
            raise ValueError("Overlapping convolution tensor")
        span += (size - 1) * stride


def _validate_conv(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None,
                   activation: str | None) -> None:
    if (x.ndim != 3 or weight.ndim != 2 or tuple(weight.shape) != (x.shape[1], 4)
            or min(x.shape) < 1 or weight.dtype != x.dtype or weight.device != x.device
            or not weight.is_contiguous()):
        raise ValueError("Expected nonempty input[B,D,T] and matching contiguous weight[D,4]")
    if bias is not None and (tuple(bias.shape) != (x.shape[1],) or bias.dtype != x.dtype
            or bias.device != x.device or not bias.is_contiguous()):
        raise ValueError("Expected a matching contiguous bias[D]")
    if activation not in (None, "silu", "swish"):
        raise ValueError("Unsupported convolution activation")
    _nonoverlapping(x)


def _prefill(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None,
             activation: str | None = None, seq_idx: torch.Tensor | None = None
             ) -> torch.Tensor:
    if seq_idx is not None:
        raise ValueError("Packed convolution sequence indices are unsupported")
    if not x.is_cuda or x.dtype != torch.bfloat16:
        output = F.conv1d(x, weight.unsqueeze(1), bias, padding=weight.shape[-1] - 1,
                          groups=x.shape[1])[:, :, :x.shape[2]]
        return F.silu(output) if activation is not None else output
    _validate_conv(x, weight, bias, activation)
    from .kernels import prefill
    return prefill(x, weight, bias, activation)


def _update(x: torch.Tensor, state: torch.Tensor, weight: torch.Tensor,
            bias: torch.Tensor | None = None, activation: str | None = None) -> torch.Tensor:
    if not x.is_cuda or x.dtype != torch.bfloat16:
        return torch_causal_conv1d_update(x, state, weight, bias, activation)
    _validate_conv(x, weight, bias, activation)
    if (x.shape[2] != 1 or tuple(state.shape) != (x.shape[0], x.shape[1], 4)
            or state.dtype != x.dtype or state.device != x.device):
        raise ValueError("Expected singleton input and matching live cache[B,D,4]")
    _nonoverlapping(state)
    pointer = state.untyped_storage().data_ptr()
    if any(v.untyped_storage().data_ptr() == pointer for v in (x, weight, bias) if v is not None):
        raise ValueError("Convolution cache must not alias input, weight or bias")
    from .kernels import update
    return update(x, state, weight, bias, activation)


def install(backbone: Qwen3_5ForConditionalGeneration, head: nn.Module | None = None
            ) -> Callable[[], None]:
    """Configure this backbone and return an idempotent cleanup function."""
    def native_cleanup() -> None:
        return None

    language = backbone.model.language_model
    if language.embed_tokens.weight.dtype != torch.bfloat16:
        return native_cleanup
    rotary = language.rotary_emb
    rope = rotary.config.rope_parameters
    if (rope.get("rope_type") != "default" or rope.get("partial_rotary_factor") != 0.25
            or rope.get("rope_theta") != 10000000.0 or rotary.config.head_dim != 256
            or rope.get("mrope_interleaved") is not True
            or list(rope.get("mrope_section", ())) != [11, 11, 10]
            or language.config.hidden_act != "silu"):
        raise ValueError("Unsupported Clairic rotary configuration")
    forwards: list[tuple[nn.Module, bool, Any, Any]] = []
    convolutions: list[tuple[Any, Any, Any]] = []
    caches: dict[torch.device, torch.Tensor] = {}
    transpose_caches: list[dict[str, Any]] = []
    attention_metadata = DecodeMetadata()
    closed = False

    def restore() -> None:
        nonlocal closed
        if closed:
            return
        closed = True
        attention_metadata.close()
        for module, original_prefill, original_update in reversed(convolutions):
            module.causal_conv1d_fn, module.causal_conv1d_update = original_prefill, original_update
        for module, had_override, original, installed in reversed(forwards):
            if module.forward == installed:
                if had_override:
                    module.forward = original
                else:
                    del module.forward
        caches.clear()
        for cache in transpose_caches:
            cache.clear()
        transpose_caches.clear()

    def bind(module: nn.Module, function: Callable[..., Any], argument: str = "hidden_states",
             parameter: torch.Tensor | None = None, *, cuda_only: bool = False) -> None:
        original = module.forward

        def forward(self: nn.Module, *args: Any, **kwargs: Any) -> Any:
            value = args[0] if args else kwargs[argument]
            dtype = parameter.dtype if parameter is not None else value.dtype
            active = dtype == torch.bfloat16 and (not cuda_only or value.is_cuda)
            return function(self, *args, **kwargs) if active else original(*args, **kwargs)

        installed = MethodType(forward, module)
        forwards.append((module, "forward" in module.__dict__, original, installed))
        module.forward = installed

    def text_rotary(self: Qwen3_5TextRotaryEmbedding, x: torch.Tensor,
                    position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
        if x.device not in caches:
            dim = int(self.config.head_dim * rope["partial_rotary_factor"])
            with torch.no_grad(), torch.autocast(device_type=x.device.type, enabled=False):
                inv = 1.0 / (rope["rope_theta"] ** (
                    torch.arange(0, dim, 2, dtype=torch.float32, device=x.device) / dim))
                positions = torch.arange(self.config.max_position_embeddings,
                                         dtype=torch.float32, device=x.device)
                frequencies = torch.einsum("i,j -> ij", positions, inv)
                caches[x.device] = torch.cat((frequencies.cos(), frequencies.sin()), -1)
        positions = position_ids[0] if position_ids.ndim == 3 else position_ids
        cos, sin = caches[x.device][positions].chunk(2, -1)
        return torch.cat((cos, cos), -1), torch.cat((sin, sin), -1)

    try:
        for module in language.modules():
            if isinstance(module, Qwen3_5RMSNorm):
                bind(module, _norm, "x", module.weight)
            elif isinstance(module, Qwen3_5DecoderLayer):
                bind(module, _decoder, parameter=module.input_layernorm.weight)
            elif isinstance(module, Qwen3_5MLP):
                bind(module, _mlp, "x", module.gate_proj.weight)
            elif isinstance(module, Qwen3_5Attention):
                def dense(self: Qwen3_5Attention, *args: Any, **kwargs: Any) -> Any:
                    return _dense(self, *args, metadata=attention_metadata, **kwargs)
                bind(module, dense, parameter=module.q_proj.weight)
            elif isinstance(module, Qwen3_5GatedDeltaNet):
                bind(module, _gdn, parameter=module.in_proj_qkv.weight, cuda_only=True)
                bind(module.norm, _gated_norm, "x", module.norm.weight, cuda_only=True)
                convolutions.append((module, module.causal_conv1d_fn, module.causal_conv1d_update))
                module.causal_conv1d_fn, module.causal_conv1d_update = _prefill, _update
        bind(rotary, text_rotary, "x")
        original_language_forward = language.forward

        def language_forward(self: nn.Module, *args: Any, **kwargs: Any) -> Any:
            with attention_metadata.scope(args, kwargs), recurrent_metadata_scope(self):
                return original_language_forward(*args, **kwargs)

        installed_language = MethodType(language_forward, language)
        forwards.append((language, "forward" in language.__dict__, original_language_forward, installed_language))
        language.forward = installed_language
        lm_head = backbone.get_output_embeddings()
        roots = (backbone,) if head is None else (backbone, head)
        seen: set[int] = set()
        for root in roots:
            for module in root.modules():
                if id(module) in seen or (type(module) is not nn.Linear and module is not lm_head):
                    continue
                seen.add(id(module))
                if module.weight.dtype not in (torch.bfloat16, torch.float32):
                    continue
                original = module.forward
                installed = MethodType(_linear_forward(original, module is lm_head, transpose_caches), module)
                forwards.append((module, "forward" in module.__dict__, original, installed))
                module.forward = installed
    except BaseException:
        restore()
        raise
    return restore
