# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright (c) 2024, Tri Dao.
# Copyright 2026 Cloudflare, Inc.

"""CUDA kernels for attention and causal convolution."""

from typing import Any

import torch
import triton
import triton.language as tl


@triton.jit(do_not_specialize=["XB", "XD", "XT", "OB", "OD", "OT"])
def _prefill(X: tl.tensor, W: tl.tensor, Bias: tl.tensor, Out: tl.tensor,
             T: tl.int32, D: tl.constexpr,
             XB: tl.int64, XD: tl.int64, XT: tl.int64,
             OB: tl.int64, OD: tl.int64, OT: tl.int64,
             HAS_BIAS: tl.constexpr, SILU: tl.constexpr,
             BT: tl.constexpr, BD: tl.constexpr) -> None:
    batch = tl.program_id(1)
    token = tl.program_id(0) * BT + tl.arange(0, BT)
    channel = tl.program_id(2) * BD + tl.arange(0, BD)
    if HAS_BIAS:
        initial = tl.load(Bias + channel, channel < D, other=0).to(tl.float32)
    else:
        initial = tl.full((BD,), 0, tl.float32)
    acc = tl.broadcast_to(initial[None, :], (BT, BD))
    for tap in tl.static_range(4):
        position = token + tap - 3
        value = tl.load(X + batch * XB + channel[None, :] * XD + position[:, None] * XT,
                        (position[:, None] >= 0) & (position[:, None] < T)
                        & (channel[None, :] < D), other=0)
        weight = tl.load(W + channel * 4 + tap, channel < D, other=0)
        acc += value * weight[None, :]
    if SILU:
        acc = acc / (1 + tl.exp(-acc))
    tl.store(Out + batch * OB + channel[None, :] * OD + token[:, None] * OT,
             acc, (token[:, None] < T) & (channel[None, :] < D))


@triton.jit(do_not_specialize=["XB", "XD", "SB", "SD", "ST", "OB", "OD"])
def _update(X: tl.tensor, State: tl.tensor, W: tl.tensor, Bias: tl.tensor, Out: tl.tensor,
            D: tl.constexpr, XB: tl.int64, XD: tl.int64,
            SB: tl.int64, SD: tl.int64, ST: tl.int64,
            OB: tl.int64, OD: tl.int64,
            HAS_BIAS: tl.constexpr, SILU: tl.constexpr, BD: tl.constexpr) -> None:
    batch = tl.program_id(0)
    channel = tl.program_id(1) * BD + tl.arange(0, BD)
    mask = channel < D
    base = State + batch * SB + channel * SD
    x0 = tl.load(base + ST, mask, other=0)
    x1 = tl.load(base + 2 * ST, mask, other=0)
    x2 = tl.load(base + 3 * ST, mask, other=0)
    x3 = tl.load(X + batch * XB + channel * XD, mask, other=0)
    if HAS_BIAS:
        acc = tl.load(Bias + channel, mask, other=0).to(tl.float32)
    else:
        acc = tl.full((BD,), 0, tl.float32)
    for tap in tl.static_range(4):
        if tap == 0:
            value = x0
        elif tap == 1:
            value = x1
        elif tap == 2:
            value = x2
        else:
            value = x3
        weight = tl.load(W + channel * 4 + tap, mask, other=0)
        acc += value * weight
    if SILU:
        acc = acc / (1 + tl.exp(-acc))
    tl.store(Out + batch * OB + channel * OD, acc, mask)
    # Preserve the four-token cache owned by Transformers.
    tl.store(base, x0, mask)
    tl.store(base + ST, x1, mask)
    tl.store(base + 2 * ST, x2, mask)
    tl.store(base + 3 * ST, x3, mask)


def prefill(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None,
            activation: str | None) -> torch.Tensor:
    output = torch.empty_like(x)
    batch, channels, length = x.shape
    _prefill[(triton.cdiv(length, 8), batch, triton.cdiv(channels, 256))](
        x, weight, bias, output, length, channels, *x.stride(), *output.stride(),
        HAS_BIAS=bias is not None, SILU=activation is not None, BT=8, BD=256,
        num_warps=4, num_stages=2,
    )
    return output


def update(x: torch.Tensor, state: torch.Tensor, weight: torch.Tensor,
           bias: torch.Tensor | None, activation: str | None) -> torch.Tensor:
    output = torch.empty_like(x)
    _update[(x.shape[0], triton.cdiv(x.shape[1], 256))](
        x, state, weight, bias, output, x.shape[1], *x.stride()[:2],
        *state.stride(), *output.stride()[:2], HAS_BIAS=bias is not None,
        SILU=activation is not None, BD=256, num_warps=4,
    )
    return output


@triton.jit(do_not_specialize=["QGT", "KT", "QT", "KOT", "GT", "CT", "ST"])
def _dense_qk_kernel(QG: tl.tensor, K: tl.tensor, QO: tl.tensor, KO: tl.tensor, GO: tl.tensor,
            QW: tl.tensor, KW: tl.tensor, C: tl.tensor, S: tl.tensor,
            QGT: tl.int64, KT: tl.int64, QT: tl.int64, KOT: tl.int64, GT: tl.int64,
            CT: tl.int64, ST: tl.int64,
            NQ: tl.constexpr, NKV: tl.constexpr, D: tl.constexpr, R: tl.constexpr,
            HALF: tl.constexpr, HEAD_BLOCK: tl.constexpr, ROT_BLOCK: tl.constexpr,
            EPS: tl.constexpr) -> None:
    token, head = tl.program_id(0), tl.program_id(1)
    is_k = head >= NQ
    local_head = tl.where(is_k, head - NQ, head)
    if is_k:
        in_base = K + token * KT + local_head * D
        w_ptr = KW
        out_base = KO + token * KOT + local_head * D
    else:
        in_base = QG + token * QGT + local_head * 2 * D
        w_ptr = QW
        out_base = QO + token * QT + local_head * D
    head_offs = tl.arange(0, HEAD_BLOCK)
    head_mask = head_offs < D
    x = tl.load(in_base + head_offs, mask=head_mask, other=0.0).to(tl.float32)
    w = tl.load(w_ptr + head_offs, mask=head_mask, other=0.0).to(tl.float32)
    var = tl.sum(x * x, axis=0) / D
    inv_rms = tl.rsqrt(var + EPS)
    x_norm = (x * inv_rms * (w + 1.0)).to(tl.bfloat16).to(tl.float32)
    pass_mask = head_mask & (head_offs >= R)
    tl.store(out_base + head_offs, x_norm, mask=pass_mask)
    rot_offs = tl.arange(0, ROT_BLOCK)
    rot_mask = rot_offs < HALF
    xr1 = tl.load(in_base + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
    xr2 = tl.load(in_base + HALF + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
    wr1 = tl.load(w_ptr + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
    wr2 = tl.load(w_ptr + HALF + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
    xr1 = (xr1 * inv_rms * (wr1 + 1.0)).to(tl.bfloat16).to(tl.float32)
    xr2 = (xr2 * inv_rms * (wr2 + 1.0)).to(tl.bfloat16).to(tl.float32)
    cos = tl.load(C + token * CT + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
    sin = tl.load(S + token * ST + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
    tl.store(out_base + rot_offs, xr1 * cos - xr2 * sin, mask=rot_mask)
    tl.store(out_base + HALF + rot_offs, xr2 * cos + xr1 * sin, mask=rot_mask)
    gate_in = in_base + D
    if not is_k:
        gate_out = GO + token * GT + local_head * D
        gate = tl.load(gate_in + head_offs, mask=head_mask, other=0.0)
        tl.store(gate_out + head_offs, gate, mask=head_mask)


def dense_qk(q_gate: torch.Tensor, key: torch.Tensor, q_weight: torch.Tensor,
        k_weight: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor,
        eps: float) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
    batch, length, q_width = q_gate.shape
    dim, rotary_dim = q_weight.numel(), cos.shape[-1]
    q_heads, kv_heads = q_width // (2 * dim), key.shape[-1] // dim
    tokens = batch * length
    qg, k = q_gate.view(tokens, -1), key.view(tokens, -1)
    c, s = cos.view(tokens, rotary_dim), sin.view(tokens, rotary_dim)
    q_out = torch.empty(tokens, q_heads * dim, device=q_gate.device, dtype=q_gate.dtype)
    k_out = torch.empty(tokens, kv_heads * dim, device=key.device, dtype=key.dtype)
    gate = torch.empty(tokens, q_heads * dim, device=q_gate.device, dtype=q_gate.dtype)
    _dense_qk_kernel[(tokens, q_heads + kv_heads)](
        qg, k, q_out, k_out, gate, q_weight, k_weight, c, s,
        qg.stride(0), k.stride(0), q_out.stride(0), k_out.stride(0), gate.stride(0),
        c.stride(0), s.stride(0), NQ=q_heads, NKV=kv_heads, D=dim, R=rotary_dim,
        HALF=rotary_dim // 2, HEAD_BLOCK=triton.next_power_of_2(dim),
        ROT_BLOCK=triton.next_power_of_2(rotary_dim // 2), EPS=eps, num_warps=4)
    return (q_out.view(batch, length, q_heads, dim).transpose(1, 2),
            k_out.view(batch, length, kv_heads, dim).transpose(1, 2),
            gate.view(batch, length, q_heads * dim))


@triton.jit
def _layer_norm_fwd_1pass_kernel(
    X: tl.tensor, Y: tl.tensor, W: tl.tensor, B: tl.tensor, Z: tl.tensor,
    Mean: tl.tensor, Rstd: tl.tensor,
    stride_x_row: tl.int64, stride_y_row: tl.int64, stride_z_row: tl.int64,
    M: tl.int32, N: tl.constexpr, eps: tl.float32,
    BLOCK_N: tl.constexpr, ROWS_PER_BLOCK: tl.constexpr,
    HAS_BIAS: tl.constexpr, HAS_Z: tl.constexpr, NORM_BEFORE_GATE: tl.constexpr,
    IS_RMS_NORM: tl.constexpr, ACTIVATION: tl.constexpr,
    USE_GDC: tl.constexpr = False,
) -> None:
    if USE_GDC:
        tl.extra.cuda.gdc_wait()
    row_start = tl.program_id(0) * ROWS_PER_BLOCK
    group = tl.program_id(1)
    rows = row_start + tl.arange(0, ROWS_PER_BLOCK)
    cols = tl.arange(0, BLOCK_N)
    row_offsets = rows[:, None] * stride_x_row
    col_offsets = cols[None, :] + group * N
    X_base = X + row_offsets + col_offsets
    Y_base = Y + rows[:, None] * stride_y_row + col_offsets
    row_mask = rows[:, None] < M
    col_mask = cols[None, :] < N
    mask = row_mask & col_mask
    x = tl.load(X_base, mask=mask, other=0.0).to(tl.float32)
    if HAS_Z and (not NORM_BEFORE_GATE):
        Z_base = Z + rows[:, None] * stride_z_row + col_offsets
        z = tl.load(Z_base, mask=mask, other=0.0).to(tl.float32)
        if ACTIVATION == 'swish' or ACTIVATION == 'silu':
            x *= z * tl.sigmoid(z)
        elif ACTIVATION == 'sigmoid':
            x *= tl.sigmoid(z)
    if not IS_RMS_NORM:
        mean = tl.sum(x, axis=1) / N
        mean_offsets = group * M + rows
        mean_mask = rows < M
        tl.store(Mean + mean_offsets, mean, mask=mean_mask)
        xbar = tl.where(mask, x - mean[:, None], 0.0)
        var = tl.sum(xbar * xbar, axis=1) / N
    else:
        xbar = tl.where(mask, x, 0.0)
        var = tl.sum(xbar * xbar, axis=1) / N
        mean = 0.0
    rstd = tl.rsqrt(var + eps)
    rstd_offsets = group * M + rows
    rstd_mask = rows < M
    tl.store(Rstd + rstd_offsets, rstd, mask=rstd_mask)
    w_offsets = cols + group * N
    w_mask = cols < N
    w = tl.load(W + w_offsets, mask=w_mask, other=0.0).to(tl.float32)
    if HAS_BIAS:
        b = tl.load(B + w_offsets, mask=w_mask, other=0.0).to(tl.float32)
    if not IS_RMS_NORM:
        x_hat = (x - mean[:, None]) * rstd[:, None]
    else:
        x_hat = x * rstd[:, None]
    y = x_hat * w[None, :] + b[None, :] if HAS_BIAS else x_hat * w[None, :]
    if HAS_Z and NORM_BEFORE_GATE:
        Z_base = Z + rows[:, None] * stride_z_row + col_offsets
        z = tl.load(Z_base, mask=mask, other=0.0).to(tl.float32)
        if ACTIVATION == 'swish' or ACTIVATION == 'silu':
            y *= z * tl.sigmoid(z)
        elif ACTIVATION == 'sigmoid':
            y *= tl.sigmoid(z)
    tl.store(Y_base, y, mask=mask)
    if USE_GDC:
        tl.extra.cuda.gdc_launch_dependents()


def gated_norm(x: torch.Tensor, z: torch.Tensor, weight: torch.Tensor,
        eps: float) -> torch.Tensor:
    output = torch.empty_like(x)
    rstd = torch.empty((x.shape[0],), dtype=torch.float32, device=x.device)
    block_n = triton.next_power_of_2(x.shape[1])
    pdl = torch.cuda.get_device_capability(x.device)[0] >= 9
    launch = {"USE_GDC": True, "launch_pdl": True} if pdl else {}
    _layer_norm_fwd_1pass_kernel[(triton.cdiv(x.shape[0], 4), 1)](
        x, output, weight, None, z, None, rstd,
        x.stride(0), output.stride(0), z.stride(0), x.shape[0], x.shape[1], eps,
        BLOCK_N=block_n, ROWS_PER_BLOCK=4, HAS_BIAS=False, HAS_Z=True,
        NORM_BEFORE_GATE=True, IS_RMS_NORM=True, ACTIVATION="swish",
        num_warps=min(max(block_n // 256, 1), 8), **launch,
    )
    return output


@triton.jit
def _fused_sigmoid_mul_kernel(
    output_ptr: tl.tensor, attn_output_ptr: tl.tensor, gate_ptr: tl.tensor,
    gate_stride_row: tl.int64, gate_stride_head: tl.int64,
    hidden_dim: tl.constexpr, HEAD_DIM: tl.constexpr, BLOCK_H: tl.constexpr,
) -> None:
    """Fuse sigmoid(gate) * attn_output into a single kernel."""
    pid_row = tl.program_id(0).to(tl.int64)
    pid_block = tl.program_id(1)
    offsets = pid_block * BLOCK_H + tl.arange(0, BLOCK_H)
    mask = offsets < hidden_dim
    head = offsets // HEAD_DIM
    d = offsets - head * HEAD_DIM
    attn_off = pid_row * hidden_dim + offsets
    attn = tl.load(attn_output_ptr + attn_off, mask=mask, other=0.0).to(tl.float32)
    gate_off = pid_row * gate_stride_row + head * gate_stride_head + d
    g = tl.load(gate_ptr + gate_off, mask=mask, other=0.0).to(tl.float32)
    result = attn * tl.sigmoid(g)
    tl.store(output_ptr + attn_off, result, mask=mask)

def fused_sigmoid_mul(attn_output: torch.Tensor, gate: torch.Tensor, inplace: bool=False) -> torch.Tensor:
    """
    Fused sigmoid-mul for attention output gating.

    Equivalent to: attn_output * sigmoid(gate)

    The production Qwen3.5 path passes a 3D strided gate. A single hidden-block
    Triton kernel handles both that path and flat contiguous inputs.

    When inplace=True, writes result back to attn_output and returns it.

    Supports strided gate: if gate is 3D (num_tokens, num_heads, head_dim)
    and attn_output is 2D (num_tokens, hidden_dim), the kernel reads gate
    via explicit strides without requiring a contiguous copy.
    """
    if gate.ndim == 3 and attn_output.ndim == 2:
        num_tokens, num_heads, head_dim = gate.shape
        hidden_dim = num_heads * head_dim
        assert attn_output.shape == (num_tokens, hidden_dim)
        gate_stride_row = gate.stride(0)
        gate_stride_head = gate.stride(1)
    else:
        assert attn_output.shape == gate.shape, 'attn_output and gate must have the same shape'
        hidden_dim = attn_output.shape[-1]
        num_tokens = attn_output.numel() // hidden_dim
        head_dim = hidden_dim
        gate_stride_row = hidden_dim
        gate_stride_head = hidden_dim
    out = attn_output if inplace else torch.empty_like(attn_output)
    block_h = 1024 if num_tokens < 1024 else 2048
    grid = (num_tokens, triton.cdiv(hidden_dim, block_h))
    _fused_sigmoid_mul_kernel[grid](out, attn_output, gate, gate_stride_row, gate_stride_head, hidden_dim, HEAD_DIM=head_dim, BLOCK_H=block_h, num_warps=4)
    return out


@triton.jit
def _silu_mul_kernel(
    gate_ptr: tl.tensor, up_ptr: tl.tensor, output_ptr: tl.tensor,
    elements: tl.constexpr, BLOCK: tl.constexpr,
) -> None:
    offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
    mask = offsets < elements
    gate = tl.load(gate_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
    up = tl.load(up_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
    # Preserve the qualified CUDA --use_fast_math operations, including FTZ.
    output = tl.inline_asm_elementwise(
        "{ .reg .f32 exponent, denominator, activated; "
        "mul.ftz.f32 exponent, $1, 0fBFB8AA3B; "
        "ex2.approx.ftz.f32 exponent, exponent; "
        "add.ftz.f32 denominator, exponent, 0f3F800000; "
        "div.approx.ftz.f32 activated, $1, denominator; "
        "mul.ftz.f32 $0, $2, activated; }",
        constraints="=f,f,f", args=[gate, up], dtype=tl.float32,
        is_pure=True, pack=1,
    )
    tl.store(output_ptr + offsets, output, mask=mask)


def silu_mul(gate: torch.Tensor, up: torch.Tensor) -> torch.Tensor:
    """Match the qualified fused CUDA SiLU-mul before the BF16 output store."""
    if (gate.shape != up.shape or gate.device != up.device
            or gate.dtype != torch.bfloat16 or up.dtype != torch.bfloat16
            or not gate.is_cuda or not gate.is_contiguous() or not up.is_contiguous()):
        raise ValueError("Expected aligned contiguous CUDA BF16 gate and up tensors")
    output = torch.empty_like(gate)
    _silu_mul_kernel[(triton.cdiv(gate.numel(), 2048),)](
        gate, up, output, gate.numel(), BLOCK=2048, num_warps=8,
    )
    return output


# MIT License
#
# Copyright 2025 Thinking Machines Lab
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

def get_device_core_count() -> int:
    return torch.cuda.get_device_properties(torch.cuda.current_device()).multi_processor_count

@triton.jit
def _compute_pid(tile_id: tl.tensor, num_pid_in_group: tl.tensor, num_pid_m: tl.tensor, GROUP_SIZE_M: tl.tensor, NUM_SMS: tl.tensor) -> tuple[tl.tensor, tl.tensor]:
    group_id = tile_id // num_pid_in_group
    first_pid_m = group_id * GROUP_SIZE_M
    group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
    pid_m = first_pid_m + tile_id % group_size_m
    pid_n = tile_id % num_pid_in_group // group_size_m
    return (pid_m, pid_n)

@triton.jit
def matmul_kernel_persistent(a_ptr: tl.tensor, b_ptr: tl.tensor, c_ptr: tl.tensor, bias_ptr: tl.tensor, M: tl.int32, N: tl.int32, K: tl.int32, stride_am: tl.int32, stride_ak: tl.int32, stride_bk: tl.int32, stride_bn: tl.int32, stride_cm: tl.int32, stride_cn: tl.int32, BLOCK_SIZE_M: tl.constexpr, BLOCK_SIZE_N: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, GROUP_SIZE_M: tl.constexpr, NUM_SMS: tl.constexpr, A_LARGE: tl.constexpr, B_LARGE: tl.constexpr, C_LARGE: tl.constexpr, HAS_BIAS: tl.constexpr) -> None:
    start_pid = tl.program_id(axis=0)
    num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
    num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
    k_tiles = tl.cdiv(K, BLOCK_SIZE_K)
    num_tiles = num_pid_m * num_pid_n
    offs_k_for_mask = tl.arange(0, BLOCK_SIZE_K)
    num_pid_in_group = GROUP_SIZE_M * num_pid_n
    for tile_id in tl.range(start_pid, num_tiles, NUM_SMS, flatten=True):
        (pid_m, pid_n) = _compute_pid(tile_id, num_pid_in_group, num_pid_m, GROUP_SIZE_M, NUM_SMS)
        start_m = pid_m * BLOCK_SIZE_M
        start_n = pid_n * BLOCK_SIZE_N
        offs_am = start_m + tl.arange(0, BLOCK_SIZE_M)
        offs_bn = start_n + tl.arange(0, BLOCK_SIZE_N)
        if A_LARGE:
            offs_am = offs_am.to(tl.int64)
        if B_LARGE:
            offs_bn = offs_bn.to(tl.int64)
        offs_am = tl.where(offs_am < M, offs_am, 0)
        offs_bn = tl.where(offs_bn < N, offs_bn, 0)
        offs_am = tl.max_contiguous(tl.multiple_of(offs_am, BLOCK_SIZE_M), BLOCK_SIZE_M)
        offs_bn = tl.max_contiguous(tl.multiple_of(offs_bn, BLOCK_SIZE_N), BLOCK_SIZE_N)
        accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
        for ki in range(k_tiles):
            if A_LARGE or B_LARGE:
                offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K).to(tl.int64)
            else:
                offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K)
            a_ptrs = a_ptr + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak)
            b_ptrs = b_ptr + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)
            a = tl.load(a_ptrs, mask=offs_k_for_mask[None, :] < K - ki * BLOCK_SIZE_K, other=0.0)
            b = tl.load(b_ptrs, mask=offs_k_for_mask[:, None] < K - ki * BLOCK_SIZE_K, other=0.0)
            accumulator = tl.dot(a, b, accumulator)
        offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
        offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
        if C_LARGE:
            offs_cm = offs_cm.to(tl.int64)
            offs_cn = offs_cn.to(tl.int64)
        c_ptrs = c_ptr + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :]
        c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N)
        if HAS_BIAS:
            bias_ptrs = bias_ptr + offs_cn
            bias = tl.load(bias_ptrs, mask=offs_cn < N, other=0.0).to(tl.float32)
            accumulator += bias
        if c_ptr.dtype.element_ty == tl.float8e4nv:
            c = accumulator.to(tl.float8e4nv)
        elif c_ptr.dtype.element_ty == tl.bfloat16:
            c = accumulator.to(tl.bfloat16)
        elif c_ptr.dtype.element_ty == tl.float32:
            c = accumulator.to(tl.float32)
        else:
            c = accumulator.to(tl.float16)
        tl.store(c_ptrs, c, mask=c_mask)

def _matmul_persistent_triton(a: torch.Tensor, b: torch.Tensor, bias: torch.Tensor | None=None) -> torch.Tensor:
    assert a.shape[1] == b.shape[0], 'Incompatible dimensions'
    assert a.dtype == b.dtype, 'Incompatible dtypes'
    assert bias is None or bias.dim() == 1, 'Expected a one-dimensional bias'
    NUM_SMS = get_device_core_count()
    (M, K) = a.shape
    (K, N) = b.shape
    dtype = a.dtype
    c = torch.empty((M, N), device=a.device, dtype=dtype)

    def grid(META: dict[str, Any]) -> tuple[int]:
        return (min(NUM_SMS, triton.cdiv(M, META['BLOCK_SIZE_M']) * triton.cdiv(N, META['BLOCK_SIZE_N'])),)
    configs = {torch.bfloat16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float32: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}}
    matmul_kernel_persistent[grid](a, b, c, bias, M, N, K, a.stride(0), a.stride(1), b.stride(0), b.stride(1), c.stride(0), c.stride(1), NUM_SMS=NUM_SMS, A_LARGE=a.numel() > 2 ** 31, B_LARGE=b.numel() > 2 ** 31, C_LARGE=c.numel() > 2 ** 31, HAS_BIAS=bias is not None, **configs[dtype])
    return c

@triton.jit
def mean_kernel(input_ptr: tl.tensor, output_ptr: tl.tensor, input_stride0: tl.int32, input_stride1: tl.int32, input_stride2: tl.int32, output_stride0: tl.int32, output_stride1: tl.int32, M: tl.int32, N: tl.int32, K: tl.int32, BLOCK_SIZE: tl.constexpr) -> None:
    """
    Kernel for computing mean along a single dimension.
    Input is viewed as (M, N, K) where N is the dimension being reduced.
    """
    pid = tl.program_id(0)
    m_idx = pid // K
    k_idx = pid % K
    if m_idx >= M or k_idx >= K:
        return
    acc = 0.0
    for n_start in range(0, N, BLOCK_SIZE):
        n_offsets = n_start + tl.arange(0, BLOCK_SIZE)
        mask = n_offsets < N
        input_idx = m_idx * input_stride0 + n_offsets * input_stride1 + k_idx * input_stride2
        vals = tl.load(input_ptr + input_idx, mask=mask, other=0.0)
        acc += tl.sum(vals)
    mean_val = acc / N
    output_idx = m_idx * output_stride0 + k_idx * output_stride1
    tl.store(output_ptr + output_idx, mean_val)

def mean_dim(input: torch.Tensor, dim: int, keepdim: bool=False, dtype: torch.dtype | None=None) -> torch.Tensor:
    """
    Triton implementation of torch.mean with single dimension reduction.

    Args:
        input: Input tensor
        dim: Single dimension along which to compute mean
        keepdim: Whether to keep the reduced dimension
        dtype: Output dtype. If None, uses input dtype (or float32 for integer inputs)

    Returns:
        Tensor with mean values along specified dimension
    """
    assert input.is_cuda or input.is_xpu, 'Input must be a CUDA or XPU tensor'
    assert -input.ndim <= dim < input.ndim, f'Invalid dimension {dim} for tensor with {input.ndim} dimensions'
    if dim < 0:
        dim = dim + input.ndim
    if dtype is None:
        if input.dtype in [torch.int8, torch.int16, torch.int32, torch.int64]:
            dtype = torch.float32
        else:
            dtype = input.dtype
    if input.dtype != dtype:
        input = input.to(dtype)
    shape = list(input.shape)
    M = 1
    for i in range(dim):
        M *= shape[i]
    N = shape[dim]
    K = 1
    for i in range(dim + 1, len(shape)):
        K *= shape[i]
    input_3d = input.reshape(M, N, K)
    if keepdim:
        output_shape = shape.copy()
        output_shape[dim] = 1
    else:
        output_shape = shape[:dim] + shape[dim + 1:]
    output = torch.empty(output_shape, dtype=dtype, device=input.device)
    if keepdim:
        output_2d = output.reshape(M, 1, K).squeeze(1)
    else:
        output_2d = output.reshape(M, K)
    grid = (M * K,)
    BLOCK_SIZE = 1024
    mean_kernel[grid](input_3d, output_2d, input_3d.stride(0), input_3d.stride(1), input_3d.stride(2), output_2d.stride(0), output_2d.stride(1) if output_2d.ndim > 1 else 0, M, N, K, BLOCK_SIZE)
    return output

def mean_batch_invariant(input: torch.Tensor, dim: list[int], keepdim: bool=False, dtype: torch.dtype | None=None) -> torch.Tensor:
    assert dtype is None or dtype == torch.float32, f'unsupported dtype: {dtype}'
    if len(dim) == 1:
        return mean_dim(input, dim[0], keepdim=keepdim)
    else:
        assert input.dtype in {torch.float16, torch.bfloat16, torch.float32}, 'only float types supported for now'
        n_elems = 1
        for d in dim:
            n_elems *= input.shape[d]
        return torch.sum(input, dim=dim, keepdim=keepdim, dtype=torch.float32) / n_elems

@triton.jit
def bmm_kernel_persistent(a_ptr: tl.tensor, b_ptr: tl.tensor, c_ptr: tl.tensor, B: tl.int32, M: tl.int32, N: tl.int32, K: tl.int32, stride_ab: tl.int32, stride_am: tl.int32, stride_ak: tl.int32, stride_bb: tl.int32, stride_bk: tl.int32, stride_bn: tl.int32, stride_cb: tl.int32, stride_cm: tl.int32, stride_cn: tl.int32, BLOCK_SIZE_M: tl.constexpr, BLOCK_SIZE_N: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, GROUP_SIZE_M: tl.constexpr, NUM_SMS: tl.constexpr, A_LARGE: tl.constexpr, B_LARGE: tl.constexpr, C_LARGE: tl.constexpr) -> None:
    """
    Batched matrix multiplication kernel that processes batches in parallel.
    Each tile processes a (BLOCK_SIZE_M, BLOCK_SIZE_N) output block for a specific batch.
    """
    start_pid = tl.program_id(axis=0)
    num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
    num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
    k_tiles = tl.cdiv(K, BLOCK_SIZE_K)
    num_tiles_per_batch = num_pid_m * num_pid_n
    num_tiles_total = B * num_tiles_per_batch
    offs_k_for_mask = tl.arange(0, BLOCK_SIZE_K)
    num_pid_in_group = GROUP_SIZE_M * num_pid_n
    for tile_id in tl.range(start_pid, num_tiles_total, NUM_SMS, flatten=True):
        batch_idx = tile_id // num_tiles_per_batch
        tile_in_batch = tile_id % num_tiles_per_batch
        (pid_m, pid_n) = _compute_pid(tile_in_batch, num_pid_in_group, num_pid_m, GROUP_SIZE_M, NUM_SMS)
        start_m = pid_m * BLOCK_SIZE_M
        start_n = pid_n * BLOCK_SIZE_N
        offs_am = start_m + tl.arange(0, BLOCK_SIZE_M)
        offs_bn = start_n + tl.arange(0, BLOCK_SIZE_N)
        if A_LARGE:
            offs_am = offs_am.to(tl.int64)
        if B_LARGE:
            offs_bn = offs_bn.to(tl.int64)
        offs_am = tl.where(offs_am < M, offs_am, 0)
        offs_bn = tl.where(offs_bn < N, offs_bn, 0)
        offs_am = tl.max_contiguous(tl.multiple_of(offs_am, BLOCK_SIZE_M), BLOCK_SIZE_M)
        offs_bn = tl.max_contiguous(tl.multiple_of(offs_bn, BLOCK_SIZE_N), BLOCK_SIZE_N)
        if A_LARGE or B_LARGE:
            batch_idx_typed = batch_idx.to(tl.int64)
        else:
            batch_idx_typed = batch_idx
        accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
        for ki in range(k_tiles):
            if A_LARGE or B_LARGE:
                offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K).to(tl.int64)
            else:
                offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K)
            a_ptrs = a_ptr + (batch_idx_typed * stride_ab + offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak)
            b_ptrs = b_ptr + (batch_idx_typed * stride_bb + offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)
            a = tl.load(a_ptrs, mask=offs_k_for_mask[None, :] < K - ki * BLOCK_SIZE_K, other=0.0)
            b = tl.load(b_ptrs, mask=offs_k_for_mask[:, None] < K - ki * BLOCK_SIZE_K, other=0.0)
            accumulator = tl.dot(a, b, accumulator)
        offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
        offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
        if C_LARGE:
            offs_cm = offs_cm.to(tl.int64)
            offs_cn = offs_cn.to(tl.int64)
        c_ptrs = c_ptr + batch_idx_typed * stride_cb + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :]
        c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N)
        if c_ptr.dtype.element_ty == tl.float8e4nv:
            c = accumulator.to(tl.float8e4nv)
        elif c_ptr.dtype.element_ty == tl.bfloat16:
            c = accumulator.to(tl.bfloat16)
        elif c_ptr.dtype.element_ty == tl.float32:
            c = accumulator.to(tl.float32)
        else:
            c = accumulator.to(tl.float16)
        tl.store(c_ptrs, c, mask=c_mask)

def bmm_batch_invariant(a: torch.Tensor, b: torch.Tensor, *, out: torch.Tensor | None=None) -> torch.Tensor:
    if a.ndim == 3 and b.ndim == 3:
        assert a.shape[0] == b.shape[0], 'Batch sizes must match'
        assert a.shape[2] == b.shape[1], 'Incompatible dimensions'
        assert a.dtype == b.dtype, 'Incompatible dtypes'
        B = a.shape[0]
        M = a.shape[1]
        K = a.shape[2]
        N = b.shape[2]
        dtype = a.dtype
        if out is None:
            c = torch.empty((B, M, N), device=a.device, dtype=dtype)
        else:
            c = out
        NUM_SMS = get_device_core_count()
        configs = {torch.bfloat16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float32: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}}
        config = configs.get(dtype)
        if config is None:
            raise ValueError(f'Unsupported dtype {dtype} for bmm_batch_invariant. Supported dtypes are: {list(configs.keys())}')
        num_tiles_per_batch = triton.cdiv(M, config['BLOCK_SIZE_M']) * triton.cdiv(N, config['BLOCK_SIZE_N'])
        num_tiles_total = B * num_tiles_per_batch
        grid = (min(NUM_SMS, num_tiles_total),)
        bmm_kernel_persistent[grid](a, b, c, B, M, N, K, a.stride(0), a.stride(1), a.stride(2), b.stride(0), b.stride(1), b.stride(2), c.stride(0), c.stride(1), c.stride(2), NUM_SMS=NUM_SMS, A_LARGE=a.numel() > 2 ** 31, B_LARGE=b.numel() > 2 ** 31, C_LARGE=c.numel() > 2 ** 31, **config)
        return c
    else:
        raise ValueError(f'bmm_batch_invariant expects 3D tensors, got shapes {a.shape} and {b.shape}')



@triton.jit(do_not_specialize=["count"])
def _bf16_pair_kernel(value: tl.tensor, high: tl.tensor, low: tl.tensor,
                 count: tl.int32, limit: tl.constexpr, block: tl.constexpr) -> None:
    offsets = tl.program_id(0) * block + tl.arange(0, block)
    current = tl.load(value + offsets, offsets < count, other=0)
    clipped = tl.minimum(tl.maximum(current, -limit, propagate_nan=tl.PropagateNan.ALL),
                         limit, propagate_nan=tl.PropagateNan.ALL)
    upper = clipped.to(tl.bfloat16)
    lower = (current - upper.to(tl.float32)).to(tl.bfloat16)
    lower = tl.where(current == 0, upper, lower)
    tl.store(high + offsets, upper, offsets < count)
    tl.store(low + offsets, lower, offsets < count)



def bf16_pair_sum(value: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
    if not value.is_cuda or value.dtype != torch.float32 or not value.is_contiguous():
        raise ValueError("Expected contiguous CUDA FP32 residuals")
    high = torch.empty_like(value, dtype=torch.bfloat16)
    low = torch.empty_like(value, dtype=torch.bfloat16)
    count = value.numel()
    if count:
        _bf16_pair_kernel[(triton.cdiv(count, 1024),)](
            value, high, low, count, torch.finfo(torch.bfloat16).max, 1024,
            num_warps=4, enable_fp_fusion=False,
        )
    return high, low



@triton.jit(do_not_specialize=["count"])
def _residual_kernel(first: tl.tensor, second: tl.tensor, output: tl.tensor,
                     count: tl.int32, block: tl.constexpr) -> None:
    offsets = tl.program_id(0) * block + tl.arange(0, block)
    upper = tl.load(first + offsets, offsets < count, other=0).to(tl.bfloat16).to(tl.float32)
    lower = tl.load(second + offsets, offsets < count, other=0).to(tl.float32)
    tl.store(output + offsets, upper + lower, offsets < count)



def residual_sum(first: torch.Tensor, second: torch.Tensor) -> torch.Tensor:
    if (not first.is_cuda or second.device != first.device or second.shape != first.shape
            or first.dtype not in (torch.bfloat16, torch.float32)
            or second.dtype != torch.bfloat16
            or not first.is_contiguous() or not second.is_contiguous()):
        raise ValueError("Expected aligned contiguous CUDA residuals")
    output = torch.empty_like(first, dtype=torch.float32)
    count = first.numel()
    if count:
        _residual_kernel[(triton.cdiv(count, 1024),)](
            first, second, output, count, 1024, num_warps=4, enable_fp_fusion=False,
        )
    return output

