[4/N][warmup][DSv4] Migrate common attention kernels (#50176)

Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Roberto L. Castro
2026-09-08 10:25:48 -04:00
committed by GitHub
co-authored by Codex
parent f998862d46
commit 6ddbab03de
19 changed files with 2970 additions and 1264 deletions
@@ -13,7 +13,9 @@ from __future__ import annotations
import pytest
import torch
from vllm.models.common.ops import fused_q_kv_rmsnorm
from vllm.models.common.ops.fused_qk_rmsnorm import (
fused_q_kv_rmsnorm,
)
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
+34
View File
@@ -431,6 +431,40 @@ def test_dispatch_can_forward_compile_key_fields() -> None:
kernel.compile_key({"tokens": 5, "mode": 2, "extra": 1})
def test_forwarded_compile_key_fields_exclude_predicate_only_inputs() -> None:
class ForwardingKernel(VllmJitKernel["ForwardingKernel.CompileKey"]):
@dataclass(frozen=True)
class CompileKey:
mode: int
block_size: int
def dispatch( # type: ignore[override]
self,
*,
tokens: int,
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(
**compile_key_fields,
block_size=_round_up(tokens, multiple=4),
)
def get_warmup_keys(self) -> list[CompileKey]:
return self._trace_dispatch(self.dispatch)(
tokens=(1, 5),
mode=2,
max_tokens=4,
_when=lambda *, tokens, max_tokens: tokens <= max_tokens,
)
def compile(self, compile_key: CompileKey) -> None:
pass
assert ForwardingKernel().get_warmup_keys() == [
ForwardingKernel.CompileKey(mode=2, block_size=4)
]
def test_dispatch_supports_tuple_and_mapping_subscriptions() -> None:
class SubscriptKernel(VllmJitKernel["SubscriptKernel.CompileKey"]):
@dataclass(frozen=True)
+18 -3
View File
@@ -14,8 +14,8 @@ from abc import ABC, abstractmethod
from collections.abc import Callable, Iterable, Iterator, Mapping
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass
from typing import Any, Generic, TypeVar
from dataclasses import dataclass, fields
from typing import Any, Generic, TypeVar, cast
__all__ = [
"JitWarmupRegistry",
@@ -597,6 +597,16 @@ class VllmJitKernel(Generic[CompileKeyT], ABC):
predicate_trace = (
_trace_warmup_predicate(_when) if _when is not None else None
)
predicate_only_names: frozenset[str] = frozenset()
if predicate_trace is not None:
compile_key_fields = frozenset(
field.name for field in fields(cast(Any, self.CompileKey))
)
predicate_only_names = (
predicate_trace.input_names
- compile_key_dispatch_trace.input_names
- compile_key_fields
)
# Unmatched **kwargs fields also belong to the expansion space.
available_names = set(kwargs).union(
*(group.rows[0] for group in input_groups)
@@ -638,7 +648,12 @@ class VllmJitKernel(Generic[CompileKeyT], ABC):
):
continue
compile_key = compile_key_dispatch_trace.compile_key(
self.CompileKey, dispatch_values
self.CompileKey,
{
name: value
for name, value in dispatch_values.items()
if name not in predicate_only_names
},
)
compile_keys[compile_key] = None
return list(compile_keys)
@@ -15,7 +15,13 @@ from vllm.model_executor.warmup.jit_warmup import (
)
CompileKeyT = TypeVar("CompileKeyT")
LaunchSpec = tuple[tuple[int, ...], dict[str, Any]]
# ``(grid, launch_kwargs)`` or ``(grid, launch_kwargs, outputs)`` for
# self-allocating kernels. ``grid=None`` skips the launch (e.g. empty batch)
# but still returns the declared outputs.
LaunchSpec = (
tuple[tuple[int, ...] | None, dict[str, Any]]
| tuple[tuple[int, ...] | None, dict[str, Any], Any]
)
def triton_scalar_specialization_rep(value: int) -> int:
@@ -91,6 +97,7 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
kernel: ClassVar[Any]
_warming = False
_warming_compile_key: CompileKeyT | None = None
@abstractmethod
def warmup_inputs(self, compile_key: CompileKeyT) -> dict[str, Any]:
@@ -100,10 +107,12 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
def compile(self, compile_key: CompileKeyT) -> None:
inputs = self.warmup_inputs(compile_key)
self._warming = True
self._warming_compile_key = compile_key
try:
cast(Callable[..., None], self)(**inputs)
finally:
self._warming = False
self._warming_compile_key = None
@cached_property
def _kernel_arg_names(self) -> tuple[str, ...]:
@@ -119,11 +128,14 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
def launch(
self,
grid: tuple[int, ...],
grid: tuple[int, ...] | None,
inputs: Mapping[str, Any],
/,
*,
kernel: Any = None,
**kwargs: Any,
) -> Any:
kernel = kernel if kernel is not None else self.kernel
runtime_launcher = kwargs.pop("_runtime_launcher", None)
runtime_launcher_arg_count = kwargs.pop("_runtime_launcher_arg_count", 0)
for name, value in inputs.items():
@@ -131,22 +143,36 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
if target in self._kernel_arg_names and target not in kwargs:
kwargs[target] = value
if self._warming:
warmup = getattr(self.kernel, "warmup", None)
if (
self._warming_compile_key is not None
and "launch_pdl" in kwargs
and hasattr(self._warming_compile_key, "launch_pdl")
):
kwargs["launch_pdl"] = self._warming_compile_key.launch_pdl
warmup = getattr(kernel, "warmup", None)
assert warmup is not None
return warmup(grid=(1,), **kwargs)
if grid is None:
return None
if runtime_launcher is not None:
regular_args = [
kwargs.pop(name)
for name in self._kernel_arg_names[:runtime_launcher_arg_count]
]
return runtime_launcher(self.kernel, grid, *regular_args, **kwargs)
return self.kernel[grid](**kwargs)
return runtime_launcher(kernel, grid, *regular_args, **kwargs)
return kernel[grid](**kwargs)
def kernel_launcher(
call_fn: Callable[..., LaunchSpec],
) -> Callable[..., None]:
"""Launch a Triton kernel from a declarative ``__call__`` specification."""
) -> Callable[..., Any]:
"""Launch a Triton kernel from a declarative ``__call__`` specification.
``call_fn`` returns either ``(grid, launch_kwargs)`` or, when it allocates
its own outputs, ``(grid, launch_kwargs, outputs)``. The declared outputs are
returned to the caller. A ``grid`` of ``None`` skips the launch (e.g. an
empty-token batch) while still returning the outputs.
"""
signature = inspect.signature(call_fn)
@wraps(call_fn)
@@ -154,14 +180,20 @@ def kernel_launcher(
self: VllmTritonJitKernel[Any],
*args: Any,
**kwargs: Any,
) -> None:
grid, launch_kwargs = call_fn(self, *args, **kwargs)
) -> Any:
spec = call_fn(self, *args, **kwargs)
if len(spec) == 3:
grid, launch_kwargs, outputs = spec
else:
grid, launch_kwargs = spec
outputs = None
bound = signature.bind(self, *args, **kwargs)
bound.apply_defaults()
inputs = {
name: value for name, value in bound.arguments.items() if name != "self"
}
self.launch(grid, inputs, **launch_kwargs)
return outputs
return wrapper
+2 -1
View File
@@ -2,8 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Ops shared across model implementations."""
from .fused_qk_rmsnorm import fused_q_kv_rmsnorm
from .fused_qk_rmsnorm import _FUSED_Q_KV_RMSNORM_KERNEL, fused_q_kv_rmsnorm
__all__ = [
"_FUSED_Q_KV_RMSNORM_KERNEL",
"fused_q_kv_rmsnorm",
]
+192 -86
View File
@@ -1,67 +1,194 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
kernel_launcher,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import next_power_of_2
@triton.jit
def _fused_q_kv_rmsnorm_kernel(
q_ptr,
q_out_ptr,
q_weight_ptr,
q_in_stride,
q_out_stride,
kv_ptr,
kv_out_ptr,
kv_weight_ptr,
kv_in_stride,
kv_out_stride,
eps,
Q_SIZE: tl.constexpr,
KV_SIZE: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
launch_pdl: tl.constexpr,
):
# num_tokens goes on grid-x (max 2**31 - 1); task goes on grid-y.
# CUDA's grid-y/z are capped at 65535, so putting num_tokens there crashes
# the launch at max-num-batched-tokens >= 65536 with "invalid argument".
# int64: q_in_stride can be ~24K (128 heads × 192) and overflows int32
# past num_tokens ~87K under large chunked prefill.
token_idx = tl.program_id(0).to(tl.int64)
pid_task = tl.program_id(1)
class FusedQKVRMSNormKernel(VllmTritonJitKernel["FusedQKVRMSNormKernel.CompileKey"]):
@dataclass(frozen=True)
class CompileKey:
dtype: torch.dtype
q_size: int
kv_size: int
block_size: int
q_in_stride: int
q_out_stride: int
kv_in_stride: int
kv_out_stride: int
eps: float
launch_pdl: bool
if pid_task == 0:
SIZE = Q_SIZE
row_in = q_ptr + token_idx * q_in_stride
weight_ptr = q_weight_ptr
row_out = q_out_ptr + token_idx * q_out_stride
else:
SIZE = KV_SIZE
row_in = kv_ptr + token_idx * kv_in_stride
weight_ptr = kv_weight_ptr
row_out = kv_out_ptr + token_idx * kv_out_stride
@staticmethod
@triton.jit
def kernel(
q_ptr,
q_out_ptr,
q_weight_ptr,
q_in_stride,
q_out_stride,
kv_ptr,
kv_out_ptr,
kv_weight_ptr,
kv_in_stride,
kv_out_stride,
eps,
Q_SIZE: tl.constexpr,
KV_SIZE: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
launch_pdl: tl.constexpr,
):
# num_tokens goes on grid-x (max 2**31 - 1); task goes on grid-y.
# CUDA's grid-y/z are capped at 65535, so putting num_tokens there crashes
# the launch at max-num-batched-tokens >= 65536 with "invalid argument".
# int64: q_in_stride can be ~24K (128 heads × 192) and overflows int32
# past num_tokens ~87K under large chunked prefill.
token_idx = tl.program_id(0).to(tl.int64)
pid_task = tl.program_id(1)
# RMSNorm in fp32 throughout — matches csrc/layernorm_kernels.cu's
# `(scalar_t)(x * s_variance * w)` and DeepseekV4's compressor kernel, which
# keep x, rrms, and w all in fp32 and perform a single cast at store.
block = tl.arange(0, BLOCK_SIZE)
mask = block < SIZE
# The weight load does not depend on the producer's output, so issue it
# before the PDL wait: gamma streams in while the producer finishes,
# leaving a single dependent global round trip (x) after the wait.
w = tl.load(weight_ptr + block, mask=mask, other=0.0).to(tl.float32)
if pid_task == 0:
SIZE = Q_SIZE
row_in = q_ptr + token_idx * q_in_stride
weight_ptr = q_weight_ptr
row_out = q_out_ptr + token_idx * q_out_stride
else:
SIZE = KV_SIZE
row_in = kv_ptr + token_idx * kv_in_stride
weight_ptr = kv_weight_ptr
row_out = kv_out_ptr + token_idx * kv_out_stride
if launch_pdl:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
# RMSNorm in fp32 throughout — matches csrc/layernorm_kernels.cu's
# `(scalar_t)(x * s_variance * w)` and DeepseekV4's compressor kernel, which
# keep x, rrms, and w all in fp32 and perform a single cast at store.
block = tl.arange(0, BLOCK_SIZE)
mask = block < SIZE
# The weight load does not depend on the producer's output, so issue it
# before the PDL wait: gamma streams in while the producer finishes,
# leaving a single dependent global round trip (x) after the wait.
w = tl.load(weight_ptr + block, mask=mask, other=0.0).to(tl.float32)
x = tl.load(row_in + block, mask=mask, other=0.0).to(tl.float32)
variance = tl.sum(x * x, axis=0) / SIZE
rrms = tl.rsqrt(variance + eps)
y = x * rrms * w
tl.store(row_out + block, y.to(row_out.dtype.element_ty), mask=mask)
if launch_pdl:
tl.extra.cuda.gdc_wait()
tl.extra.cuda.gdc_launch_dependents()
x = tl.load(row_in + block, mask=mask, other=0.0).to(tl.float32)
variance = tl.sum(x * x, axis=0) / SIZE
rrms = tl.rsqrt(variance + eps)
y = x * rrms * w
tl.store(row_out + block, y.to(row_out.dtype.element_ty), mask=mask)
def dispatch( # type: ignore[override]
self,
*,
dtype: torch.dtype,
q_size: int,
kv_size: int,
eps: float,
launch_pdl: bool,
**compile_key_fields: int,
) -> CompileKey:
max_size = q_size if q_size >= kv_size else kv_size
return self.CompileKey(
**compile_key_fields,
dtype=dtype,
q_size=q_size,
kv_size=kv_size,
block_size=next_power_of_2(max_size),
eps=eps,
launch_pdl=launch_pdl,
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
hf_config = vllm_config.model_config.hf_config
q_size = int(getattr(hf_config, "q_lora_rank", 0) or 0)
kv_size = int(getattr(hf_config, "head_dim", 0) or 0)
if q_size <= 0 or kv_size <= 0:
return []
input_stride = q_size + kv_size
return self._trace_dispatch(self.dispatch)(
dtype=vllm_config.model_config.dtype,
q_size=q_size,
kv_size=kv_size,
q_in_stride=input_stride,
q_out_stride=(input_stride, q_size),
kv_in_stride=input_stride,
kv_out_stride=(input_stride, kv_size),
eps=float(hf_config.rms_norm_eps),
launch_pdl=(False, True),
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
return dict(
q=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.q_size),
strides=(compile_key.q_in_stride, 1),
),
kv=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.kv_size),
strides=(compile_key.kv_in_stride, 1),
),
q_out=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.q_size),
strides=(compile_key.q_out_stride, 1),
),
kv_out=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.kv_size),
strides=(compile_key.kv_out_stride, 1),
),
q_weight=TritonWarmupTensor(
compile_key.dtype,
shape=(compile_key.q_size,),
),
kv_weight=TritonWarmupTensor(
compile_key.dtype,
shape=(compile_key.kv_size,),
),
eps=compile_key.eps,
)
@kernel_launcher
def __call__(
self,
q: torch.Tensor,
kv: torch.Tensor,
q_out: torch.Tensor,
kv_out: torch.Tensor,
q_weight: torch.Tensor,
kv_weight: torch.Tensor,
eps: float,
) -> LaunchSpec:
q_size = q.shape[1]
kv_size = kv.shape[1]
block_size = next_power_of_2(max(q_size, kv_size))
return (q.shape[0], 2), dict(
q_in_stride=q.stride(0),
q_out_stride=q_out.stride(0),
kv_in_stride=kv.stride(0),
kv_out_stride=kv_out.stride(0),
Q_SIZE=q_size,
KV_SIZE=kv_size,
BLOCK_SIZE=block_size,
launch_pdl=current_platform.is_arch_support_pdl(),
# One vectorized 128-bit access per thread for K3/DSv4-sized rows,
# instead of a serial per-thread chain at the default 4 warps.
num_warps=8 if block_size >= 2048 else 4,
)
def fused_q_kv_rmsnorm(
@@ -78,40 +205,19 @@ def fused_q_kv_rmsnorm(
assert qr.stride(-1) == 1 and kv.stride(-1) == 1
assert q_weight.is_contiguous() and kv_weight.is_contiguous()
q_size = qr.shape[1]
kv_size = kv.shape[1]
num_tokens = qr.shape[0]
# Allocate packed outputs rather than empty_like: qr/kv are typically
# column slices of a fused qkv_a projection, and for num_tokens == 1
# empty_like preserves the parent's row stride (torch keeps strides of
# size-1 dims), e.g. (2112, 1) for a [1, 1536] q slice. Downstream
# dispatchers that require packed row-major inputs (e.g. the Kimi-K3
# low-latency GEMM's _is_packed_row_major check) then reject the tensor
# and silently fall back to cuBLAS on every decode step.
qr_out = torch.empty(qr.shape, dtype=qr.dtype, device=qr.device)
kv_out = torch.empty(kv.shape, dtype=kv.dtype, device=kv.device)
if num_tokens == 0:
return qr_out, kv_out
block_size = triton.next_power_of_2(max(q_size, kv_size))
_fused_q_kv_rmsnorm_kernel[(num_tokens, 2)](
qr,
qr_out,
q_weight,
qr.stride(0),
qr_out.stride(0),
kv,
kv_out,
kv_weight,
kv.stride(0),
kv_out.stride(0),
eps,
Q_SIZE=q_size,
KV_SIZE=kv_size,
BLOCK_SIZE=block_size,
launch_pdl=current_platform.is_arch_support_pdl(),
# One vectorized 128-bit access per thread for K3/DSv4-sized rows,
# instead of a serial per-thread chain at the default 4 warps.
num_warps=8 if block_size >= 2048 else 4,
)
if qr.shape[0] > 0:
_FUSED_Q_KV_RMSNORM_KERNEL(
qr,
kv,
qr_out,
kv_out,
q_weight,
kv_weight,
eps,
)
return qr_out, kv_out
_FUSED_Q_KV_RMSNORM_KERNEL = FusedQKVRMSNormKernel()
+9 -5
View File
@@ -39,9 +39,9 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.deepseek_mtp import SharedHead
from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weight_name
from vllm.model_executor.models.utils import maybe_prefix
from vllm.models.deepseek_v4.common.ops import (
fused_mtp_input_rmsnorm,
mtp_shared_head_rmsnorm,
from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import (
_FUSED_MTP_INPUT_RMSNORM_KERNEL,
_MTP_SHARED_HEAD_RMSNORM_KERNEL,
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
@@ -127,6 +127,10 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
self.hc_head_op = HCHeadOp()
if vllm_config.kernel_config.enable_jit_warmup:
_FUSED_MTP_INPUT_RMSNORM_KERNEL.register_warmup()
_MTP_SHARED_HEAD_RMSNORM_KERNEL.register_warmup()
def forward(
self,
input_ids: torch.Tensor,
@@ -143,7 +147,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
-1, self.hc_mult, self.config.hidden_size
)
# Fused: mask inputs at position 0 (not needed by MTP), enorm, hnorm.
inputs_embeds, previous_hidden_states = fused_mtp_input_rmsnorm(
inputs_embeds, previous_hidden_states = _FUSED_MTP_INPUT_RMSNORM_KERNEL(
inputs_embeds,
positions,
previous_hidden_states,
@@ -256,7 +260,7 @@ class DeepSeekV4MultiTokenPredictor(nn.Module):
mtp_layer.rms_norm_eps,
mtp_layer.hc_eps,
)
hidden_states = mtp_shared_head_rmsnorm(
hidden_states = _MTP_SHARED_HEAD_RMSNORM_KERNEL(
hidden_states,
mtp_layer.shared_head.norm.weight.data,
mtp_layer.shared_head.norm.variance_epsilon,
+58 -1
View File
@@ -22,7 +22,10 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.models.common.ops import fused_q_kv_rmsnorm
from vllm.models.common.ops.fused_qk_rmsnorm import (
_FUSED_Q_KV_RMSNORM_KERNEL,
fused_q_kv_rmsnorm,
)
from vllm.models.deepseek_v4.common.ops import (
fused_indexer_q_rope_quant,
)
@@ -417,6 +420,44 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
_COMBINE_TOPK_SWA_INDICES_KERNEL.register_warmup()
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_cutedsl
_FUSED_Q_KV_RMSNORM_KERNEL.register_warmup()
backend_name = self.backend_cls.get_name()
if current_platform.is_cuda():
from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import ( # noqa: E501
_FUSED_INV_ROPE_FP8_QUANT_KERNEL,
)
_FUSED_INV_ROPE_FP8_QUANT_KERNEL.register_warmup()
if self.compress_ratio == 128:
from vllm.models.deepseek_v4.sparse_mla import (
_BUILD_C128A_TOPK_METADATA_KERNEL,
)
_BUILD_C128A_TOPK_METADATA_KERNEL.register_warmup()
if backend_name == "FLASHMLA_SPARSE_DSV4":
from vllm.models.deepseek_v4.common.ops.cache_utils import (
_COMPUTE_GLOBAL_TOPK_INDICES_AND_LENS_KERNEL,
_DEQUANTIZE_AND_GATHER_K_CACHE_KERNEL,
)
_COMPUTE_GLOBAL_TOPK_INDICES_AND_LENS_KERNEL.register_warmup()
if not has_cutedsl():
_DEQUANTIZE_AND_GATHER_K_CACHE_KERNEL.register_warmup()
elif backend_name == "FLASHINFER_MLA_SPARSE_DSV4":
from vllm.models.deepseek_v4.common.ops.cache_utils import (
_BUILD_FLASHINFER_MIXED_SPARSE_INDICES_KERNEL,
_COMPUTE_GLOBAL_TOPK_INDICES_AND_LENS_KERNEL,
)
_COMPUTE_GLOBAL_TOPK_INDICES_AND_LENS_KERNEL.register_warmup()
_BUILD_FLASHINFER_MIXED_SPARSE_INDICES_KERNEL.register_warmup()
def forward(
self,
positions: torch.Tensor,
@@ -974,6 +1015,22 @@ class DeepseekV4Indexer(nn.Module):
torch.cuda.Event(),
]
if vllm_config.kernel_config.enable_jit_warmup:
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_cutedsl
if not has_cutedsl() and not current_platform.is_xpu():
from vllm.models.deepseek_v4.common.ops.fused_indexer_q import (
_FUSED_INDEXER_Q_ROPE_MXFP4_TRITON_KERNEL,
_FUSED_INDEXER_Q_ROPE_QUANT_TRITON_KERNEL,
)
(
_FUSED_INDEXER_Q_ROPE_MXFP4_TRITON_KERNEL
if self.use_fp4_kv
else _FUSED_INDEXER_Q_ROPE_QUANT_TRITON_KERNEL
).register_warmup()
def forward(
self,
hidden_states: torch.Tensor,
@@ -10,8 +10,6 @@ from .cache_utils import (
)
from .fused_indexer_q import MXFP4_BLOCK_SIZE, fused_indexer_q_rope_quant
from .fused_inv_rope_fp8_quant import fused_inv_rope_fp8_quant
from .fused_mtp_input_rmsnorm import fused_mtp_input_rmsnorm, mtp_shared_head_rmsnorm
from .save_partial_states import save_partial_states
__all__ = [
"MXFP4_BLOCK_SIZE",
@@ -21,8 +19,5 @@ __all__ = [
"dequantize_and_gather_k_cache",
"fused_indexer_q_rope_quant",
"fused_inv_rope_fp8_quant",
"fused_mtp_input_rmsnorm",
"mtp_shared_head_rmsnorm",
"quantize_and_insert_k_cache",
"save_partial_states",
]
File diff suppressed because it is too large Load Diff
@@ -19,13 +19,22 @@ even/odd halves, producing (N_QUANT_BLOCKS, MXFP4_BLOCK/2) packed nibbles
and N_QUANT_BLOCKS ue8m0 bytes.
"""
from dataclasses import dataclass
from functools import lru_cache
from types import SimpleNamespace
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
kernel_launcher,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import round_up
if current_platform.is_rocm():
from vllm.platforms.rocm import _ON_GFX950
@@ -68,14 +77,32 @@ def compress_norm_rope_store_triton(
kernel = _fused_kv_compress_norm_rope_insert_sparse_attn
num_warps = 4
kernel_kwargs = {"SANITIZE_CACHE_NANS": _ON_GFX950}
elif use_fp4_cache:
kernel = _fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn
num_warps = 1
kernel_kwargs = {}
else:
kernel = _fused_kv_compress_norm_rope_insert_indexer_attn
num_warps = 1
kernel_kwargs = {}
_FUSED_KV_COMPRESS_NORM_ROPE_INSERT_INDEXER_TRITON_KERNEL(
state_cache=state_cache,
num_actual=num_actual,
token_to_req_indices=token_to_req_indices,
positions=positions,
slot_mapping=slot_mapping,
block_table=block_table,
block_size=block_size,
state_width=state_width,
cos_sin_cache=cos_sin_cache,
kv_cache=kv_cache,
k_cache_metadata=k_cache_metadata,
pdl_kwargs=pdl_kwargs,
head_dim=head_dim,
rope_head_dim=rope_head_dim,
compress_ratio=compress_ratio,
overlap=overlap,
use_fp4_cache=use_fp4_cache,
rms_norm_weight=rms_norm_weight,
rms_norm_eps=rms_norm_eps,
quant_block=quant_block,
token_stride=token_stride,
scale_dim=scale_dim,
)
return
kernel[(num_actual,)](
# state cache
@@ -673,7 +700,15 @@ def compress_norm_rope_store_two_stage_triton(
# =============================================================================
# Indexer path (head=128, all FP8, single quant block)
# =============================================================================
@triton.jit
@triton.jit(
do_not_specialize=[
"state_cache_stride0",
"state_cache_stride1",
"block_table_stride",
"rms_norm_eps",
"cos_sin_stride",
]
)
def _fused_kv_compress_norm_rope_insert_indexer_attn(
# ── state cache (compressor internal state) ──
state_cache_ptr,
@@ -850,7 +885,15 @@ def _fused_kv_compress_norm_rope_insert_indexer_attn(
# =============================================================================
# Indexer path (head=128, MXFP4: 2 nibbles/byte + ue8m0 per 32-elem block)
# =============================================================================
@triton.jit
@triton.jit(
do_not_specialize=[
"state_cache_stride0",
"state_cache_stride1",
"block_table_stride",
"rms_norm_eps",
"cos_sin_stride",
]
)
def _fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn(
# ── state cache (compressor internal state) ──
state_cache_ptr,
@@ -1038,3 +1081,232 @@ def _fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn(
tl.store(val_ptr + tl.arange(0, TOKEN_STRIDE), packed_flat)
tl.store(scale_ptr + tl.arange(0, SCALE_DIM), ue8m0)
class FusedKVCompressNormRopeInsertIndexerTritonKernel(
VllmTritonJitKernel["FusedKVCompressNormRopeInsertIndexerTritonKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
dtype: torch.dtype
use_fp4_cache: bool
head_size: int
triton_block_size: int
state_width: int
compress_ratio: int
overlap: bool
rope_head_dim: int
fp8_max: float
quant_block: int
token_stride: int
scale_dim: int
block_size: int
kv_cache_block_size: int
kv_block_stride: int
# Canonical kernel for the launcher's arg_names mapping; the launched
# variant (fp8/mxfp4, same arg_names) is picked per call in __call__.
kernel = staticmethod(_fused_kv_compress_norm_rope_insert_indexer_attn)
def dispatch( # type: ignore[override]
self,
*,
dtype: torch.dtype,
use_fp4_cache: bool,
head_dim: int,
rope_head_dim: int,
compress_ratio: int,
cache_block_size: int,
cache_alignment: int,
runtime_state_width: int | None = None,
runtime_quant_block: int | None = None,
runtime_token_stride: int | None = None,
runtime_scale_dim: int | None = None,
runtime_kv_block_stride: int | None = None,
) -> CompileKey:
overlap = compress_ratio == 4
default_quant_block = 32 if use_fp4_cache else 128
quant_block = (
runtime_quant_block
if runtime_quant_block is not None
else default_quant_block
)
default_token_stride = head_dim // 2 if use_fp4_cache else head_dim
token_stride = (
runtime_token_stride
if runtime_token_stride is not None
else default_token_stride
)
default_scale_dim = head_dim // quant_block if use_fp4_cache else 4
scale_dim = (
runtime_scale_dim if runtime_scale_dim is not None else default_scale_dim
)
raw_kv_cache_block_size = cache_block_size // compress_ratio
kv_cache_block_size = (
raw_kv_cache_block_size if raw_kv_cache_block_size >= 1 else 1
)
state_block_size = (
4
if compress_ratio == 4
else 8
if compress_ratio == 128
else cache_block_size
)
default_kv_block_stride = round_up(
kv_cache_block_size * (token_stride + scale_dim),
cache_alignment,
)
kv_block_stride = (
runtime_kv_block_stride
if runtime_kv_block_stride is not None
else default_kv_block_stride
)
state_width = (
runtime_state_width
if runtime_state_width is not None
else head_dim * (1 + overlap)
)
return self.CompileKey(
dtype=dtype,
use_fp4_cache=use_fp4_cache,
head_size=head_dim,
triton_block_size=triton.next_power_of_2(head_dim),
state_width=state_width,
compress_ratio=compress_ratio,
overlap=overlap,
rope_head_dim=rope_head_dim,
fp8_max=448.0,
quant_block=quant_block,
token_stride=token_stride,
scale_dim=scale_dim,
block_size=state_block_size,
kv_cache_block_size=kv_cache_block_size,
kv_block_stride=kv_block_stride,
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
hf_config = vllm_config.model_config.hf_config
if hf_config is None:
return []
cache_block_size = vllm_config.cache_config.block_size
from vllm.v1.attention.backends.mla.indexer import (
dsa_indexer_uses_fp4,
)
use_fp4_cache = dsa_indexer_uses_fp4(vllm_config)
head_dim = int(getattr(hf_config, "index_head_dim", 0) or 0)
rope_head_dim = int(getattr(hf_config, "qk_rope_head_dim", 0) or 0)
if head_dim <= 0 or rope_head_dim <= 0 or cache_block_size <= 0:
return []
cache_dtype = vllm_config.cache_config.cache_dtype
cache_alignment = 576 if cache_dtype == "fp8_ds_mla" else 512
return self._trace_dispatch(self.dispatch)(
dtype=vllm_config.model_config.dtype,
use_fp4_cache=use_fp4_cache,
head_dim=head_dim,
rope_head_dim=rope_head_dim,
compress_ratio=4,
cache_block_size=cache_block_size,
cache_alignment=cache_alignment,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
fp32_ptr = TritonWarmupTensor(torch.float32, shape=(1, 1))
int32_ptr = TritonWarmupTensor(torch.int32)
int64_ptr = TritonWarmupTensor(torch.int64)
# kv_cache supplies kv_cache_block_size (shape[1]) and KV_BLOCK_STRIDE
# (stride(0)); reproduce both so warmup and launch specialize identically.
kv_cache = TritonWarmupTensor(
torch.uint8,
shape=(1, compile_key.kv_cache_block_size, 1),
strides=(compile_key.kv_block_stride, 1, 1),
)
return dict(
state_cache=fp32_ptr,
num_actual=1,
token_to_req_indices=int32_ptr,
positions=int64_ptr,
slot_mapping=int64_ptr,
block_table=int32_ptr,
block_size=compile_key.block_size,
state_width=compile_key.state_width,
cos_sin_cache=fp32_ptr,
kv_cache=kv_cache,
k_cache_metadata=SimpleNamespace(slot_mapping=int64_ptr),
pdl_kwargs={},
head_dim=compile_key.head_size,
rope_head_dim=compile_key.rope_head_dim,
compress_ratio=compile_key.compress_ratio,
overlap=compile_key.overlap,
use_fp4_cache=compile_key.use_fp4_cache,
rms_norm_weight=TritonWarmupTensor(
compile_key.dtype,
shape=(compile_key.head_size,),
),
rms_norm_eps=1e-6,
quant_block=compile_key.quant_block,
token_stride=compile_key.token_stride,
scale_dim=compile_key.scale_dim,
)
@kernel_launcher
def __call__(
self,
*,
state_cache: torch.Tensor,
num_actual: int,
token_to_req_indices: torch.Tensor,
positions: torch.Tensor,
slot_mapping: torch.Tensor,
block_table: torch.Tensor,
block_size: int,
state_width: int,
cos_sin_cache: torch.Tensor,
kv_cache: torch.Tensor,
k_cache_metadata: Any,
pdl_kwargs: dict,
head_dim: int,
rope_head_dim: int,
compress_ratio: int,
overlap: bool,
use_fp4_cache: bool,
rms_norm_weight: torch.Tensor,
rms_norm_eps: float,
quant_block: int,
token_stride: int,
scale_dim: int,
) -> LaunchSpec:
return (num_actual,), dict(
kernel=(
_fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn
if use_fp4_cache
else _fused_kv_compress_norm_rope_insert_indexer_attn
),
state_cache_stride0=state_cache.stride(0),
state_cache_stride1=state_cache.stride(1),
block_table_stride=block_table.stride(0),
cos_sin_stride=cos_sin_cache.stride(0),
k_cache_ptr=kv_cache,
kv_slot_mapping_ptr=k_cache_metadata.slot_mapping,
kv_cache_block_size=kv_cache.shape[1],
HEAD_SIZE=head_dim,
TRITON_BLOCK_SIZE=triton.next_power_of_2(head_dim),
STATE_WIDTH=state_width,
COMPRESS_RATIO=compress_ratio,
OVERLAP=overlap,
ROPE_HEAD_DIM=rope_head_dim,
FP8_MAX=448.0,
QUANT_BLOCK=quant_block,
TOKEN_STRIDE=token_stride,
SCALE_DIM=scale_dim,
KV_BLOCK_STRIDE=kv_cache.stride(0),
num_warps=1,
**pdl_kwargs,
)
_FUSED_KV_COMPRESS_NORM_ROPE_INSERT_INDEXER_TRITON_KERNEL = (
FusedKVCompressNormRopeInsertIndexerTritonKernel()
)
@@ -1,8 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
kernel_launcher,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.import_utils import has_cutedsl
@@ -67,229 +76,459 @@ def _quantize_mxfp4_pair(x_lo, x_hi):
return packed, ue8m0
@triton.jit
def _fused_indexer_q_rope_quant_kernel(
pos_ptr,
# Index Q RoPE
index_q_ptr,
index_q_stride0,
index_q_stride1,
index_q_cos_sin_ptr,
index_q_cos_sin_stride,
INDEX_Q_HALF_ROT_DIM: tl.constexpr,
# Index Q Quantize
index_q_fp8_ptr,
index_q_fp8_stride0,
index_q_fp8_stride1,
INDEX_Q_HEAD_DIM: tl.constexpr,
# Index weights
index_weights_ptr,
index_weights_stride,
index_weights_softmax_scale,
index_weights_head_scale,
index_weights_out_ptr,
index_weights_out_stride,
FP8_MAX: tl.constexpr = 448.0,
USE_FNUZ: tl.constexpr = False,
USE_EXPLICIT_FMA: tl.constexpr = False,
class FusedIndexerQRopeQuantTritonKernel(
VllmTritonJitKernel["FusedIndexerQRopeQuantTritonKernel.CompileKey"]
):
# Layout matches the unfused reference (DeepseekV4ScalingRotaryEmbedding
# + per_token_group_quant_fp8): GPT-J interleaved RoPE applied to the
# LAST rope_dim dims of each head; the leading [0, NOPE_DIM) is passed
# through unchanged.
INDEX_Q_ROT_DIM: tl.constexpr = 2 * INDEX_Q_HALF_ROT_DIM
INDEX_Q_NOPE_DIM: tl.constexpr = INDEX_Q_HEAD_DIM - INDEX_Q_ROT_DIM
tl.static_assert(INDEX_Q_NOPE_DIM >= 0)
@dataclass(frozen=True)
class CompileKey:
dtype: torch.dtype
num_heads: int
index_q_half_rot_dim: int
index_q_head_dim: int
fp8_max: float
use_fnuz: bool
use_explicit_fma: bool
tok_idx = tl.program_id(0)
head_idx = tl.program_id(1)
pos = tl.load(pos_ptr + tok_idx)
cos, sin = _get_cos_sin(
@staticmethod
@triton.jit
def kernel(
pos_ptr,
# Index Q RoPE
index_q_ptr,
index_q_stride0,
index_q_stride1,
index_q_cos_sin_ptr,
index_q_cos_sin_stride,
pos,
INDEX_Q_HALF_ROT_DIM,
)
half_offset = tl.arange(0, INDEX_Q_HALF_ROT_DIM)
base_ptr = index_q_ptr + tok_idx * index_q_stride0 + head_idx * index_q_stride1
INDEX_Q_HALF_ROT_DIM: tl.constexpr,
# Index Q Quantize
index_q_fp8_ptr,
index_q_fp8_stride0,
index_q_fp8_stride1,
INDEX_Q_HEAD_DIM: tl.constexpr,
# Index weights
index_weights_ptr,
index_weights_stride,
index_weights_softmax_scale,
index_weights_head_scale,
index_weights_out_ptr,
index_weights_out_stride,
FP8_MAX: tl.constexpr = 448.0,
USE_FNUZ: tl.constexpr = False,
USE_EXPLICIT_FMA: tl.constexpr = False,
):
# Layout matches the unfused reference (DeepseekV4ScalingRotaryEmbedding
# + per_token_group_quant_fp8): GPT-J interleaved RoPE applied to the
# LAST rope_dim dims of each head; the leading [0, NOPE_DIM) is passed
# through unchanged.
INDEX_Q_ROT_DIM: tl.constexpr = 2 * INDEX_Q_HALF_ROT_DIM
INDEX_Q_NOPE_DIM: tl.constexpr = INDEX_Q_HEAD_DIM - INDEX_Q_ROT_DIM
tl.static_assert(INDEX_Q_NOPE_DIM >= 0)
# Interleaved (GPT-J) RoPE on dims [NOPE_DIM, HEAD_DIM):
# even = q[NOPE_DIM + 2*i], odd = q[NOPE_DIM + 2*i + 1]
rot_base = base_ptr + INDEX_Q_NOPE_DIM
x_even = tl.load(rot_base + half_offset * 2).to(tl.float32)
x_odd = tl.load(rot_base + half_offset * 2 + 1).to(tl.float32)
if USE_EXPLICIT_FMA:
# Match HIP rotary_embedding contraction before bf16 materialization.
r_even = tl.fma(x_even, cos, -(x_odd * sin))
r_odd = tl.fma(x_odd, cos, x_even * sin)
else:
r_even = x_even * cos - x_odd * sin
r_odd = x_odd * cos + x_even * sin
tok_idx = tl.program_id(0)
head_idx = tl.program_id(1)
# Match reference numerics: fp32 → bf16 → fp32 before the ue8m0 absmax.
# Same pattern as the K-side compressor kernel (fused_compress_quant_cache.py).
r_even = r_even.to(tl.bfloat16).to(tl.float32)
r_odd = r_odd.to(tl.bfloat16).to(tl.float32)
amax = tl.maximum(tl.max(tl.abs(r_even)), tl.max(tl.abs(r_odd)))
if INDEX_Q_NOPE_DIM > 0:
nope_offset = tl.arange(0, INDEX_Q_NOPE_DIM)
x_nope = tl.load(base_ptr + nope_offset).to(tl.float32)
amax = tl.maximum(amax, tl.max(tl.abs(x_nope)))
index_q_scale = tl.div_rn(tl.maximum(amax, 1e-4), FP8_MAX)
index_q_scale = tl.math.exp2(tl.math.ceil(tl.math.log2(index_q_scale)))
# Store quantized values to index_q_fp8. FNUZ (e4m3fnuz) on gfx942, OCP
# (e4m3fn) elsewhere -- matches the K cache.
fp8_dtype = tl.float8e4b8 if USE_FNUZ else tl.float8e4nv
fp8_base_ptr = (
index_q_fp8_ptr + tok_idx * index_q_fp8_stride0 + head_idx * index_q_fp8_stride1
)
if INDEX_Q_NOPE_DIM > 0:
tl.store(
fp8_base_ptr + nope_offset,
tl.div_rn(x_nope, index_q_scale).to(fp8_dtype),
pos = tl.load(pos_ptr + tok_idx)
cos, sin = _get_cos_sin(
index_q_cos_sin_ptr,
index_q_cos_sin_stride,
pos,
INDEX_Q_HALF_ROT_DIM,
)
fp8_rot_base = fp8_base_ptr + INDEX_Q_NOPE_DIM
tl.store(
fp8_rot_base + half_offset * 2,
tl.div_rn(r_even, index_q_scale).to(fp8_dtype),
)
tl.store(
fp8_rot_base + half_offset * 2 + 1,
tl.div_rn(r_odd, index_q_scale).to(fp8_dtype),
)
half_offset = tl.arange(0, INDEX_Q_HALF_ROT_DIM)
base_ptr = index_q_ptr + tok_idx * index_q_stride0 + head_idx * index_q_stride1
# FP8 weight-fold contract:
# index_weights_out = index_weights * q_scale * softmax_scale * head_scale
# The per-token-per-head q_scale (fp32) IS folded into the output weights
# here because FP8 Q is stored WITHOUT a companion scale tensor — the
# downstream fp8_fp4_mqa_logits/fp8_fp4_paged_mqa_logits kernels use `weights` to
# apply per-token Q scale inline. See the MXFP4 kernel below for the
# contrasting convention (scales live with the Q values, weights are NOT
# q-scaled).
index_weights = tl.load(
index_weights_ptr + tok_idx * index_weights_stride + head_idx
)
index_weights = index_weights.to(tl.float32)
index_weights *= index_q_scale
index_weights *= index_weights_softmax_scale
index_weights *= index_weights_head_scale
tl.store(
index_weights_out_ptr + tok_idx * index_weights_out_stride + head_idx,
index_weights,
)
# Interleaved (GPT-J) RoPE on dims [NOPE_DIM, HEAD_DIM):
# even = q[NOPE_DIM + 2*i], odd = q[NOPE_DIM + 2*i + 1]
rot_base = base_ptr + INDEX_Q_NOPE_DIM
x_even = tl.load(rot_base + half_offset * 2).to(tl.float32)
x_odd = tl.load(rot_base + half_offset * 2 + 1).to(tl.float32)
if USE_EXPLICIT_FMA:
# Match HIP rotary_embedding contraction before bf16 materialization.
r_even = tl.fma(x_even, cos, -(x_odd * sin))
r_odd = tl.fma(x_odd, cos, x_even * sin)
else:
r_even = x_even * cos - x_odd * sin
r_odd = x_odd * cos + x_even * sin
@triton.jit
def _fused_indexer_q_rope_mxfp4_kernel(
pos_ptr,
# Index Q RoPE input (fp/bf16)
index_q_ptr,
index_q_stride0,
index_q_stride1,
index_q_cos_sin_ptr,
index_q_cos_sin_stride,
INDEX_Q_HALF_ROT_DIM: tl.constexpr,
# MXFP4 Q outputs
index_q_mxfp4_ptr, # uint8, (T, H, HEAD_DIM // 2)
index_q_mxfp4_stride0,
index_q_mxfp4_stride1,
index_q_scale_ptr, # uint8 ue8m0, (T, H, HEAD_DIM // BLOCK)
index_q_scale_stride0,
index_q_scale_stride1,
INDEX_Q_HEAD_DIM: tl.constexpr,
MXFP4_BLOCK: tl.constexpr,
# Weights (NO per-token q_scale fold for MXFP4; per-block scales stay
# with the Q values in the output scale tensor).
index_weights_ptr,
index_weights_stride,
index_weights_softmax_scale,
index_weights_head_scale,
index_weights_out_ptr,
index_weights_out_stride,
):
INDEX_Q_ROT_DIM: tl.constexpr = 2 * INDEX_Q_HALF_ROT_DIM
INDEX_Q_NOPE_DIM: tl.constexpr = INDEX_Q_HEAD_DIM - INDEX_Q_ROT_DIM
NUM_NOPE_BLOCKS: tl.constexpr = INDEX_Q_NOPE_DIM // MXFP4_BLOCK
NUM_ROPE_BLOCKS: tl.constexpr = INDEX_Q_ROT_DIM // MXFP4_BLOCK
HALF_BLOCK: tl.constexpr = MXFP4_BLOCK // 2
tl.static_assert(INDEX_Q_NOPE_DIM >= 0)
tl.static_assert(INDEX_Q_NOPE_DIM % MXFP4_BLOCK == 0)
tl.static_assert(INDEX_Q_ROT_DIM % MXFP4_BLOCK == 0)
tl.static_assert(MXFP4_BLOCK % 2 == 0)
tok_idx = tl.program_id(0)
head_idx = tl.program_id(1)
pos = tl.load(pos_ptr + tok_idx)
q_base = index_q_ptr + tok_idx * index_q_stride0 + head_idx * index_q_stride1
out_base = (
index_q_mxfp4_ptr
+ tok_idx * index_q_mxfp4_stride0
+ head_idx * index_q_mxfp4_stride1
)
scale_base = (
index_q_scale_ptr
+ tok_idx * index_q_scale_stride0
+ head_idx * index_q_scale_stride1
)
half_off = tl.arange(0, HALF_BLOCK)
# ---- NoPE blocks: direct load, pair as (even-index, odd-index) values ----
for b in tl.static_range(NUM_NOPE_BLOCKS):
base = b * MXFP4_BLOCK
x_lo = tl.load(q_base + base + half_off * 2).to(tl.float32)
x_hi = tl.load(q_base + base + half_off * 2 + 1).to(tl.float32)
packed, ue8m0 = _quantize_mxfp4_pair(x_lo, x_hi)
tl.store(out_base + base // 2 + half_off, packed)
tl.store(scale_base + b, ue8m0)
# ---- RoPE blocks: apply GPT-J interleaved RoPE to the block's 16 pairs,
# then quantize. Each block covers HALF_BLOCK (=16) cos/sin pairs. ----
rot_q_base = q_base + INDEX_Q_NOPE_DIM
for b in tl.static_range(NUM_ROPE_BLOCKS):
pair_off = b * HALF_BLOCK + half_off # indices in [0, HALF_ROT_DIM)
cos_b = tl.load(
index_q_cos_sin_ptr + pos * index_q_cos_sin_stride + pair_off
).to(tl.float32)
sin_b = tl.load(
index_q_cos_sin_ptr
+ pos * index_q_cos_sin_stride
+ pair_off
+ INDEX_Q_HALF_ROT_DIM
).to(tl.float32)
x_even = tl.load(rot_q_base + pair_off * 2).to(tl.float32)
x_odd = tl.load(rot_q_base + pair_off * 2 + 1).to(tl.float32)
r_even = x_even * cos_b - x_odd * sin_b
r_odd = x_odd * cos_b + x_even * sin_b
# bf16 roundtrip for parity with the FP8 kernel / reference numerics.
# Match reference numerics: fp32 → bf16 → fp32 before the ue8m0 absmax.
# Same pattern as the K-side compressor kernel (fused_compress_quant_cache.py).
r_even = r_even.to(tl.bfloat16).to(tl.float32)
r_odd = r_odd.to(tl.bfloat16).to(tl.float32)
packed, ue8m0 = _quantize_mxfp4_pair(r_even, r_odd)
rope_byte_off = (INDEX_Q_NOPE_DIM + b * MXFP4_BLOCK) // 2
tl.store(out_base + rope_byte_off + half_off, packed)
tl.store(scale_base + NUM_NOPE_BLOCKS + b, ue8m0)
# MXFP4 weight-fold contract:
# index_weights_out = index_weights * softmax_scale * head_scale
# NOTE: q_scale is NOT folded here (contrast with the FP8 kernel above).
# MXFP4 Q emits a separate ue8m0 scale tensor of shape
# (T, H, HEAD_DIM // MXFP4_BLOCK) alongside the packed values, so each
# per-block scale is applied by the downstream MXFP4 logits kernel when
# dequantizing Q — there is no per-token scalar to fold into `weights`.
index_weights = tl.load(
index_weights_ptr + tok_idx * index_weights_stride + head_idx
).to(tl.float32)
index_weights *= index_weights_softmax_scale
index_weights *= index_weights_head_scale
tl.store(
index_weights_out_ptr + tok_idx * index_weights_out_stride + head_idx,
index_weights,
)
amax = tl.maximum(tl.max(tl.abs(r_even)), tl.max(tl.abs(r_odd)))
if INDEX_Q_NOPE_DIM > 0:
nope_offset = tl.arange(0, INDEX_Q_NOPE_DIM)
x_nope = tl.load(base_ptr + nope_offset).to(tl.float32)
amax = tl.maximum(amax, tl.max(tl.abs(x_nope)))
index_q_scale = tl.div_rn(tl.maximum(amax, 1e-4), FP8_MAX)
index_q_scale = tl.math.exp2(tl.math.ceil(tl.math.log2(index_q_scale)))
# Store quantized values to index_q_fp8. FNUZ (e4m3fnuz) on gfx942, OCP
# (e4m3fn) elsewhere -- matches the K cache.
fp8_dtype = tl.float8e4b8 if USE_FNUZ else tl.float8e4nv
fp8_base_ptr = (
index_q_fp8_ptr
+ tok_idx * index_q_fp8_stride0
+ head_idx * index_q_fp8_stride1
)
if INDEX_Q_NOPE_DIM > 0:
tl.store(
fp8_base_ptr + nope_offset,
tl.div_rn(x_nope, index_q_scale).to(fp8_dtype),
)
fp8_rot_base = fp8_base_ptr + INDEX_Q_NOPE_DIM
tl.store(
fp8_rot_base + half_offset * 2,
tl.div_rn(r_even, index_q_scale).to(fp8_dtype),
)
tl.store(
fp8_rot_base + half_offset * 2 + 1,
tl.div_rn(r_odd, index_q_scale).to(fp8_dtype),
)
# FP8 weight-fold contract:
# index_weights_out = index_weights * q_scale * softmax_scale * head_scale
# The per-token-per-head q_scale (fp32) IS folded into the output weights
# here because FP8 Q is stored WITHOUT a companion scale tensor — the
# downstream fp8_fp4_mqa_logits/fp8_fp4_paged_mqa_logits kernels use
# `weights` to apply per-token Q scale inline. See the MXFP4 kernel below for
# contrasting convention (scales live with the Q values, weights are NOT
# q-scaled).
index_weights = tl.load(
index_weights_ptr + tok_idx * index_weights_stride + head_idx
)
index_weights = index_weights.to(tl.float32)
index_weights *= index_q_scale
index_weights *= index_weights_softmax_scale
index_weights *= index_weights_head_scale
tl.store(
index_weights_out_ptr + tok_idx * index_weights_out_stride + head_idx,
index_weights,
)
def dispatch( # type: ignore[override]
self,
*,
dtype: torch.dtype,
num_heads: int,
head_dim: int,
rope_dim: int,
use_fnuz: bool,
) -> CompileKey:
return self.CompileKey(
dtype=dtype,
num_heads=num_heads,
index_q_half_rot_dim=rope_dim // 2,
index_q_head_dim=head_dim,
fp8_max=224.0 if use_fnuz else 448.0,
use_fnuz=use_fnuz,
use_explicit_fma=current_platform.is_rocm(),
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
hf_config = vllm_config.model_config.hf_config
num_heads = int(getattr(hf_config, "index_n_heads", 0) or 0)
head_dim = int(getattr(hf_config, "index_head_dim", 0) or 0)
rope_dim = int(getattr(hf_config, "qk_rope_head_dim", 0) or 0)
if num_heads <= 0 or head_dim <= 0 or rope_dim <= 0:
return []
return self._trace_dispatch(self.dispatch)(
dtype=vllm_config.model_config.dtype,
num_heads=num_heads,
head_dim=head_dim,
rope_dim=rope_dim,
use_fnuz=current_platform.fp8_dtype() == torch.float8_e4m3fnuz,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
q_stride0 = compile_key.num_heads * compile_key.index_q_head_dim
return dict(
positions=TritonWarmupTensor(torch.int64),
index_q=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.num_heads, compile_key.index_q_head_dim),
strides=(q_stride0, compile_key.index_q_head_dim, 1),
),
index_q_cos_sin_cache=TritonWarmupTensor(
torch.float32,
shape=(1, compile_key.index_q_half_rot_dim * 2),
),
index_weights=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.num_heads),
),
index_weights_softmax_scale=1.0,
index_weights_head_scale=1.0,
index_q_fp8=TritonWarmupTensor(
current_platform.fp8_dtype(),
shape=(1, compile_key.num_heads, compile_key.index_q_head_dim),
strides=(q_stride0, compile_key.index_q_head_dim, 1),
),
index_weights_out=TritonWarmupTensor(
torch.float32,
shape=(1, compile_key.num_heads),
),
fp8_max=compile_key.fp8_max,
use_fnuz=compile_key.use_fnuz,
)
@kernel_launcher
def __call__(
self,
positions: torch.Tensor,
index_q: torch.Tensor,
index_q_cos_sin_cache: torch.Tensor,
index_weights: torch.Tensor,
index_weights_softmax_scale: float,
index_weights_head_scale: float,
index_q_fp8: torch.Tensor,
index_weights_out: torch.Tensor,
*,
fp8_max: float,
use_fnuz: bool,
) -> LaunchSpec:
num_tokens = positions.shape[0]
num_index_q_heads = index_q.shape[1]
return (num_tokens, num_index_q_heads), dict(
pos_ptr=positions,
index_q_stride0=index_q.stride(0),
index_q_stride1=index_q.stride(1),
index_q_cos_sin_ptr=index_q_cos_sin_cache,
index_q_cos_sin_stride=index_q_cos_sin_cache.stride(0),
INDEX_Q_HALF_ROT_DIM=index_q_cos_sin_cache.shape[-1] // 2,
index_q_fp8_stride0=index_q_fp8.stride(0),
index_q_fp8_stride1=index_q_fp8.stride(1),
INDEX_Q_HEAD_DIM=index_q.shape[2],
index_weights_stride=index_weights.stride(0),
index_weights_out_stride=index_weights_out.stride(0),
FP8_MAX=fp8_max,
USE_FNUZ=use_fnuz,
USE_EXPLICIT_FMA=current_platform.is_rocm(),
num_warps=1,
)
class FusedIndexerQRopeMxFp4TritonKernel(
VllmTritonJitKernel["FusedIndexerQRopeMxFp4TritonKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
dtype: torch.dtype
num_heads: int
index_q_half_rot_dim: int
index_q_head_dim: int
mxfp4_block: int
@staticmethod
@triton.jit
def kernel(
pos_ptr,
# Index Q RoPE input (fp/bf16)
index_q_ptr,
index_q_stride0,
index_q_stride1,
index_q_cos_sin_ptr,
index_q_cos_sin_stride,
INDEX_Q_HALF_ROT_DIM: tl.constexpr,
# MXFP4 Q outputs
index_q_mxfp4_ptr, # uint8, (T, H, HEAD_DIM // 2)
index_q_mxfp4_stride0,
index_q_mxfp4_stride1,
index_q_scale_ptr, # uint8 ue8m0, (T, H, HEAD_DIM // BLOCK)
index_q_scale_stride0,
index_q_scale_stride1,
INDEX_Q_HEAD_DIM: tl.constexpr,
MXFP4_BLOCK: tl.constexpr,
# Weights (NO per-token q_scale fold for MXFP4; per-block scales stay
# with the Q values in the output scale tensor).
index_weights_ptr,
index_weights_stride,
index_weights_softmax_scale,
index_weights_head_scale,
index_weights_out_ptr,
index_weights_out_stride,
):
INDEX_Q_ROT_DIM: tl.constexpr = 2 * INDEX_Q_HALF_ROT_DIM
INDEX_Q_NOPE_DIM: tl.constexpr = INDEX_Q_HEAD_DIM - INDEX_Q_ROT_DIM
NUM_NOPE_BLOCKS: tl.constexpr = INDEX_Q_NOPE_DIM // MXFP4_BLOCK
NUM_ROPE_BLOCKS: tl.constexpr = INDEX_Q_ROT_DIM // MXFP4_BLOCK
HALF_BLOCK: tl.constexpr = MXFP4_BLOCK // 2
tl.static_assert(INDEX_Q_NOPE_DIM >= 0)
tl.static_assert(INDEX_Q_NOPE_DIM % MXFP4_BLOCK == 0)
tl.static_assert(INDEX_Q_ROT_DIM % MXFP4_BLOCK == 0)
tl.static_assert(MXFP4_BLOCK % 2 == 0)
tok_idx = tl.program_id(0)
head_idx = tl.program_id(1)
pos = tl.load(pos_ptr + tok_idx)
q_base = index_q_ptr + tok_idx * index_q_stride0 + head_idx * index_q_stride1
out_base = (
index_q_mxfp4_ptr
+ tok_idx * index_q_mxfp4_stride0
+ head_idx * index_q_mxfp4_stride1
)
scale_base = (
index_q_scale_ptr
+ tok_idx * index_q_scale_stride0
+ head_idx * index_q_scale_stride1
)
half_off = tl.arange(0, HALF_BLOCK)
# ---- NoPE blocks: direct load, pair as (even-index, odd-index) values ----
for b in tl.static_range(NUM_NOPE_BLOCKS):
base = b * MXFP4_BLOCK
x_lo = tl.load(q_base + base + half_off * 2).to(tl.float32)
x_hi = tl.load(q_base + base + half_off * 2 + 1).to(tl.float32)
packed, ue8m0 = _quantize_mxfp4_pair(x_lo, x_hi)
tl.store(out_base + base // 2 + half_off, packed)
tl.store(scale_base + b, ue8m0)
# ---- RoPE blocks: apply GPT-J interleaved RoPE to the block's 16 pairs,
# then quantize. Each block covers HALF_BLOCK (=16) cos/sin pairs. ----
rot_q_base = q_base + INDEX_Q_NOPE_DIM
for b in tl.static_range(NUM_ROPE_BLOCKS):
pair_off = b * HALF_BLOCK + half_off # indices in [0, HALF_ROT_DIM)
cos_b = tl.load(
index_q_cos_sin_ptr + pos * index_q_cos_sin_stride + pair_off
).to(tl.float32)
sin_b = tl.load(
index_q_cos_sin_ptr
+ pos * index_q_cos_sin_stride
+ pair_off
+ INDEX_Q_HALF_ROT_DIM
).to(tl.float32)
x_even = tl.load(rot_q_base + pair_off * 2).to(tl.float32)
x_odd = tl.load(rot_q_base + pair_off * 2 + 1).to(tl.float32)
r_even = x_even * cos_b - x_odd * sin_b
r_odd = x_odd * cos_b + x_even * sin_b
# bf16 roundtrip for parity with the FP8 kernel / reference numerics.
r_even = r_even.to(tl.bfloat16).to(tl.float32)
r_odd = r_odd.to(tl.bfloat16).to(tl.float32)
packed, ue8m0 = _quantize_mxfp4_pair(r_even, r_odd)
rope_byte_off = (INDEX_Q_NOPE_DIM + b * MXFP4_BLOCK) // 2
tl.store(out_base + rope_byte_off + half_off, packed)
tl.store(scale_base + NUM_NOPE_BLOCKS + b, ue8m0)
# MXFP4 weight-fold contract:
# index_weights_out = index_weights * softmax_scale * head_scale
# NOTE: q_scale is NOT folded here (contrast with the FP8 kernel above).
# MXFP4 Q emits a separate ue8m0 scale tensor of shape
# (T, H, HEAD_DIM // MXFP4_BLOCK) alongside the packed values, so each
# per-block scale is applied by the downstream MXFP4 logits kernel when
# dequantizing Q — there is no per-token scalar to fold into `weights`.
index_weights = tl.load(
index_weights_ptr + tok_idx * index_weights_stride + head_idx
).to(tl.float32)
index_weights *= index_weights_softmax_scale
index_weights *= index_weights_head_scale
tl.store(
index_weights_out_ptr + tok_idx * index_weights_out_stride + head_idx,
index_weights,
)
def dispatch( # type: ignore[override]
self,
*,
dtype: torch.dtype,
num_heads: int,
head_dim: int,
rope_dim: int,
) -> CompileKey:
return self.CompileKey(
dtype=dtype,
num_heads=num_heads,
index_q_half_rot_dim=rope_dim // 2,
index_q_head_dim=head_dim,
mxfp4_block=MXFP4_BLOCK_SIZE,
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
hf_config = vllm_config.model_config.hf_config
num_heads = int(getattr(hf_config, "index_n_heads", 0) or 0)
head_dim = int(getattr(hf_config, "index_head_dim", 0) or 0)
rope_dim = int(getattr(hf_config, "qk_rope_head_dim", 0) or 0)
if num_heads <= 0 or head_dim <= 0 or rope_dim <= 0:
return []
return self._trace_dispatch(self.dispatch)(
dtype=vllm_config.model_config.dtype,
num_heads=num_heads,
head_dim=head_dim,
rope_dim=rope_dim,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
q_stride0 = compile_key.num_heads * compile_key.index_q_head_dim
packed_head_dim = compile_key.index_q_head_dim // 2
scale_head_dim = compile_key.index_q_head_dim // compile_key.mxfp4_block
return dict(
positions=TritonWarmupTensor(torch.int64),
index_q=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.num_heads, compile_key.index_q_head_dim),
strides=(q_stride0, compile_key.index_q_head_dim, 1),
),
index_q_cos_sin_cache=TritonWarmupTensor(
torch.float32,
shape=(1, compile_key.index_q_half_rot_dim * 2),
),
index_weights=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.num_heads),
),
index_weights_softmax_scale=1.0,
index_weights_head_scale=1.0,
index_q_packed=TritonWarmupTensor(
torch.uint8,
shape=(1, compile_key.num_heads, packed_head_dim),
),
index_q_scale=TritonWarmupTensor(
torch.uint8,
shape=(1, compile_key.num_heads, scale_head_dim),
),
index_weights_out=TritonWarmupTensor(
torch.float32,
shape=(1, compile_key.num_heads),
),
)
@kernel_launcher
def __call__(
self,
positions: torch.Tensor,
index_q: torch.Tensor,
index_q_cos_sin_cache: torch.Tensor,
index_weights: torch.Tensor,
index_weights_softmax_scale: float,
index_weights_head_scale: float,
index_q_packed: torch.Tensor,
index_q_scale: torch.Tensor,
index_weights_out: torch.Tensor,
) -> LaunchSpec:
num_tokens = positions.shape[0]
num_index_q_heads = index_q.shape[1]
return (num_tokens, num_index_q_heads), dict(
pos_ptr=positions,
index_q_stride0=index_q.stride(0),
index_q_stride1=index_q.stride(1),
index_q_cos_sin_ptr=index_q_cos_sin_cache,
index_q_cos_sin_stride=index_q_cos_sin_cache.stride(0),
INDEX_Q_HALF_ROT_DIM=index_q_cos_sin_cache.shape[-1] // 2,
index_q_mxfp4_ptr=index_q_packed,
index_q_mxfp4_stride0=index_q_packed.stride(0),
index_q_mxfp4_stride1=index_q_packed.stride(1),
index_q_scale_stride0=index_q_scale.stride(0),
index_q_scale_stride1=index_q_scale.stride(1),
INDEX_Q_HEAD_DIM=index_q.shape[2],
MXFP4_BLOCK=MXFP4_BLOCK_SIZE,
index_weights_stride=index_weights.stride(0),
index_weights_out_stride=index_weights_out.stride(0),
num_warps=1,
)
def fused_indexer_q_rope_quant(
@@ -386,29 +625,16 @@ def fused_indexer_q_rope_quant(
index_weights_out,
)
else:
_fused_indexer_q_rope_mxfp4_kernel[(num_tokens, num_index_q_heads)](
_FUSED_INDEXER_Q_ROPE_MXFP4_TRITON_KERNEL(
positions,
index_q,
index_q.stride(0),
index_q.stride(1),
index_q_cos_sin_cache,
index_q_cos_sin_cache.stride(0),
index_q_cos_sin_cache.shape[-1] // 2,
index_q_packed,
index_q_packed.stride(0),
index_q_packed.stride(1),
index_q_scale,
index_q_scale.stride(0),
index_q_scale.stride(1),
index_q_head_dim,
MXFP4_BLOCK_SIZE,
index_weights,
index_weights.stride(0),
index_weights_softmax_scale,
index_weights_head_scale,
index_q_packed,
index_q_scale,
index_weights_out,
index_weights_out.stride(0),
num_warps=1, # TODO: Tune this
)
# Values stay uint8 (2 E2M1 nibbles per byte). Scales are 4 ue8m0
@@ -453,27 +679,20 @@ def fused_indexer_q_rope_quant(
index_weights_out,
)
else:
_fused_indexer_q_rope_quant_kernel[(num_tokens, num_index_q_heads)](
_FUSED_INDEXER_Q_ROPE_QUANT_TRITON_KERNEL(
positions,
index_q,
index_q.stride(0),
index_q.stride(1),
index_q_cos_sin_cache,
index_q_cos_sin_cache.stride(0),
index_q_cos_sin_cache.shape[-1] // 2,
index_q_fp8,
index_q_fp8.stride(0),
index_q_fp8.stride(1),
index_q_head_dim,
index_weights,
index_weights.stride(0),
index_weights_softmax_scale,
index_weights_head_scale,
index_q_fp8,
index_weights_out,
index_weights_out.stride(0),
FP8_MAX=fp8_max,
USE_FNUZ=use_fnuz,
USE_EXPLICIT_FMA=current_platform.is_rocm(),
num_warps=1, # TODO: Tune this
fp8_max=fp8_max,
use_fnuz=use_fnuz,
)
return index_q_fp8, index_weights_out
_FUSED_INDEXER_Q_ROPE_QUANT_TRITON_KERNEL = FusedIndexerQRopeQuantTritonKernel()
_FUSED_INDEXER_Q_ROPE_MXFP4_TRITON_KERNEL = FusedIndexerQRopeMxFp4TritonKernel()
@@ -7,145 +7,327 @@ Output scale format is pre-transformed (MN-major TMA-aligned; FP32 on SM90,
INT32-packed UE8M0 on SM100) so fp8_einsum skips transform_sf_into_required_layout.
"""
from dataclasses import dataclass
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
kernel_launcher,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
@triton.jit(do_not_specialize=["num_tokens"])
def _fused_inv_rope_fp8_quant_per_head(
o_ptr,
positions_ptr,
cos_sin_cache_ptr,
fp8_ptr,
scale_ptr,
num_tokens,
heads_per_group: tl.constexpr,
o_stride_token,
o_stride_head,
cache_stride_pos,
fp8_stride_group,
fp8_stride_token,
scale_stride_group,
scale_stride_k,
fp8_max: tl.constexpr,
eps: tl.constexpr,
QUANT_GROUP_SIZE: tl.constexpr,
CHUNKS_PER_HEAD: tl.constexpr,
ROPE_START: tl.constexpr,
HALF_ROPE: tl.constexpr,
TMA_ALIGNED_SCALES: tl.constexpr,
USE_GDC: tl.constexpr,
launch_pdl: tl.constexpr, # triton metadata
class FusedInvRopeFP8QuantKernel(
VllmTritonJitKernel["FusedInvRopeFP8QuantKernel.CompileKey"]
):
# Cast every stride to int64 — without this, Python-int strides are
# inferred as int32 and `pid_token(int64) × stride(int32)` can lower to
# int32 arithmetic, wrapping past 2³¹ for large prefill batches → IMA.
pid_token = tl.program_id(0).to(tl.int64)
pid_gh = tl.program_id(1).to(tl.int64)
o_stride_token = o_stride_token.to(tl.int64)
o_stride_head = o_stride_head.to(tl.int64)
cache_stride_pos = cache_stride_pos.to(tl.int64)
fp8_stride_group = fp8_stride_group.to(tl.int64)
fp8_stride_token = fp8_stride_token.to(tl.int64)
scale_stride_group = scale_stride_group.to(tl.int64)
scale_stride_k = scale_stride_k.to(tl.int64)
@dataclass(frozen=True)
class CompileKey:
heads_per_group: int
fp8_max: float
quant_group_size: int
chunks_per_head: int
rope_start: int
half_rope: int
tma_aligned_scales: bool
launch_pdl: bool
g = pid_gh // heads_per_group
head_in_group = pid_gh % heads_per_group
global_head = pid_gh
qb_start = head_in_group * CHUNKS_PER_HEAD
if USE_GDC:
tl.extra.cuda.gdc_launch_dependents()
tl.extra.cuda.gdc_wait()
# Padding rows in the TMA-aligned scale buffer: fill with zero and skip quant.
if pid_token >= num_tokens:
@staticmethod
# scale_stride_k = align(num_tokens, 4) is only 4-aligned, so its Triton int
# class varies with batch size; not specialized so one warmup key covers all
# (int64-cast, scalar addressing only, so perf-neutral).
@triton.jit(do_not_specialize=["num_tokens", "scale_stride_k"])
def kernel(
o_ptr,
positions_ptr,
cos_sin_cache_ptr,
fp8_ptr,
scale_ptr,
num_tokens,
heads_per_group: tl.constexpr,
o_stride_token,
o_stride_head,
cache_stride_pos,
fp8_stride_group,
fp8_stride_token,
scale_stride_group,
scale_stride_k,
fp8_max: tl.constexpr,
eps: tl.constexpr,
QUANT_GROUP_SIZE: tl.constexpr,
CHUNKS_PER_HEAD: tl.constexpr,
ROPE_START: tl.constexpr,
HALF_ROPE: tl.constexpr,
TMA_ALIGNED_SCALES: tl.constexpr,
launch_pdl: tl.constexpr,
):
# Cast every stride to int64 — without this, Python-int strides are
# inferred as int32 and `pid_token(int64) × stride(int32)` can lower to
# int32 arithmetic, wrapping past 2³¹ for large prefill batches → IMA.
pid_token = tl.program_id(0).to(tl.int64)
pid_gh = tl.program_id(1).to(tl.int64)
o_stride_token = o_stride_token.to(tl.int64)
o_stride_head = o_stride_head.to(tl.int64)
cache_stride_pos = cache_stride_pos.to(tl.int64)
fp8_stride_group = fp8_stride_group.to(tl.int64)
fp8_stride_token = fp8_stride_token.to(tl.int64)
scale_stride_group = scale_stride_group.to(tl.int64)
scale_stride_k = scale_stride_k.to(tl.int64)
g = pid_gh // heads_per_group
head_in_group = pid_gh % heads_per_group
global_head = pid_gh
qb_start = head_in_group * CHUNKS_PER_HEAD
if launch_pdl:
tl.extra.cuda.gdc_launch_dependents()
tl.extra.cuda.gdc_wait()
# Padding rows in the TMA-aligned scale buffer: fill with zero and skip quant.
if pid_token >= num_tokens:
if TMA_ALIGNED_SCALES:
scale_addr = (
scale_ptr
+ g * scale_stride_group
+ pid_token
+ head_in_group * scale_stride_k
)
tl.store(scale_addr, tl.zeros((), dtype=tl.int32))
else:
block_offsets = tl.arange(0, CHUNKS_PER_HEAD)
qb_indices = qb_start + block_offsets
scale_addrs = (
scale_ptr
+ g * scale_stride_group
+ pid_token
+ qb_indices * scale_stride_k
)
tl.store(scale_addrs, tl.zeros((CHUNKS_PER_HEAD,), dtype=tl.float32))
return
input_base = o_ptr + pid_token * o_stride_token + global_head * o_stride_head
HEAD_DIM: tl.constexpr = CHUNKS_PER_HEAD * QUANT_GROUP_SIZE
offsets = tl.arange(0, HEAD_DIM)
x = tl.load(input_base + offsets).to(tl.float32)
rope_abs_start: tl.constexpr = (
CHUNKS_PER_HEAD - 1
) * QUANT_GROUP_SIZE + ROPE_START
pos = tl.load(positions_ptr + pid_token)
cache_base = cos_sin_cache_ptr + pos * cache_stride_pos
is_rope = offsets >= rope_abs_start
rope_local = offsets - rope_abs_start
x_partner = tl.load(input_base + (offsets ^ 1), mask=is_rope, other=0.0).to(
tl.float32
)
cs_idx = tl.maximum(rope_local >> 1, 0)
cos_v = tl.load(cache_base + cs_idx, mask=is_rope, other=1.0)
sin_v = tl.load(cache_base + HALF_ROPE + cs_idx, mask=is_rope, other=0.0)
x_add = x * cos_v + x_partner * sin_v
x_sub = x * cos_v - x_partner * sin_v
is_even = (rope_local & 1) == 0
rotated = tl.where(is_even, x_add, x_sub)
x = tl.where(is_rope, rotated, x)
x_2d = tl.reshape(tl.abs(x), (CHUNKS_PER_HEAD, QUANT_GROUP_SIZE))
block_absmax = tl.maximum(tl.max(x_2d, axis=1), eps)
scale_raw = block_absmax * (1.0 / fp8_max)
scales = tl.math.exp2(tl.ceil(tl.log2(scale_raw)))
scales_exp = tl.reshape(
tl.broadcast_to(
tl.reshape(scales, (CHUNKS_PER_HEAD, 1)),
(CHUNKS_PER_HEAD, QUANT_GROUP_SIZE),
),
(HEAD_DIM,),
)
x_quant = tl.clamp(x / scales_exp, -fp8_max, fp8_max).to(tl.float8e4nv)
fp8_base = (
fp8_ptr
+ g * fp8_stride_group
+ pid_token * fp8_stride_token
+ qb_start * QUANT_GROUP_SIZE
)
tl.store(fp8_base + offsets, x_quant)
block_offsets = tl.arange(0, CHUNKS_PER_HEAD)
qb_indices = qb_start + block_offsets
if TMA_ALIGNED_SCALES:
scale_bits = scales.to(tl.int32, bitcast=True)
ue8m0_bytes = (scale_bits >> 23) & 0xFF
packed_val = tl.sum(ue8m0_bytes << (block_offsets * 8))
scale_addr = (
scale_ptr
+ g * scale_stride_group
+ pid_token
+ head_in_group * scale_stride_k
)
tl.store(scale_addr, tl.zeros((), dtype=tl.int32))
tl.store(scale_addr, packed_val)
else:
block_offsets = tl.arange(0, CHUNKS_PER_HEAD)
qb_indices = qb_start + block_offsets
scale_addrs = (
scale_ptr
+ g * scale_stride_group
+ pid_token
+ qb_indices * scale_stride_k
)
tl.store(scale_addrs, tl.zeros((CHUNKS_PER_HEAD,), dtype=tl.float32))
return
tl.store(scale_addrs, scales)
input_base = o_ptr + pid_token * o_stride_token + global_head * o_stride_head
HEAD_DIM: tl.constexpr = CHUNKS_PER_HEAD * QUANT_GROUP_SIZE
offsets = tl.arange(0, HEAD_DIM)
x = tl.load(input_base + offsets).to(tl.float32)
rope_abs_start: tl.constexpr = (CHUNKS_PER_HEAD - 1) * QUANT_GROUP_SIZE + ROPE_START
pos = tl.load(positions_ptr + pid_token)
cache_base = cos_sin_cache_ptr + pos * cache_stride_pos
is_rope = offsets >= rope_abs_start
rope_local = offsets - rope_abs_start
x_partner = tl.load(input_base + (offsets ^ 1), mask=is_rope, other=0.0).to(
tl.float32
)
cs_idx = tl.maximum(rope_local >> 1, 0)
cos_v = tl.load(cache_base + cs_idx, mask=is_rope, other=1.0)
sin_v = tl.load(cache_base + HALF_ROPE + cs_idx, mask=is_rope, other=0.0)
x_add = x * cos_v + x_partner * sin_v
x_sub = x * cos_v - x_partner * sin_v
is_even = (rope_local & 1) == 0
rotated = tl.where(is_even, x_add, x_sub)
x = tl.where(is_rope, rotated, x)
x_2d = tl.reshape(tl.abs(x), (CHUNKS_PER_HEAD, QUANT_GROUP_SIZE))
block_absmax = tl.maximum(tl.max(x_2d, axis=1), eps)
scale_raw = block_absmax * (1.0 / fp8_max)
scales = tl.math.exp2(tl.ceil(tl.log2(scale_raw)))
scales_exp = tl.reshape(
tl.broadcast_to(
tl.reshape(scales, (CHUNKS_PER_HEAD, 1)),
(CHUNKS_PER_HEAD, QUANT_GROUP_SIZE),
),
(HEAD_DIM,),
)
x_quant = tl.clamp(x / scales_exp, -fp8_max, fp8_max).to(tl.float8e4nv)
fp8_base = (
fp8_ptr
+ g * fp8_stride_group
+ pid_token * fp8_stride_token
+ qb_start * QUANT_GROUP_SIZE
)
tl.store(fp8_base + offsets, x_quant)
block_offsets = tl.arange(0, CHUNKS_PER_HEAD)
qb_indices = qb_start + block_offsets
if TMA_ALIGNED_SCALES:
scale_bits = scales.to(tl.int32, bitcast=True)
ue8m0_bytes = (scale_bits >> 23) & 0xFF
packed_val = tl.sum(ue8m0_bytes << (block_offsets * 8))
scale_addr = (
scale_ptr
+ g * scale_stride_group
+ pid_token
+ head_in_group * scale_stride_k
def dispatch( # type: ignore[override]
self,
*,
heads_per_group: int,
head_dim: int,
nope_dim: int,
rope_dim: int,
quant_group_size: int,
runtime_fp8_max: float | None = None,
runtime_chunks_per_head: int | None = None,
runtime_rope_start: int | None = None,
runtime_half_rope: int | None = None,
**compile_key_fields: bool,
) -> CompileKey:
fp8_max = (
runtime_fp8_max
if runtime_fp8_max is not None
else torch.finfo(torch.float8_e4m3fn).max
)
tl.store(scale_addr, packed_val)
else:
scale_addrs = (
scale_ptr + g * scale_stride_group + pid_token + qb_indices * scale_stride_k
chunks_per_head = (
runtime_chunks_per_head
if runtime_chunks_per_head is not None
else head_dim // quant_group_size
)
rope_start = (
runtime_rope_start
if runtime_rope_start is not None
else nope_dim % quant_group_size
)
half_rope = (
runtime_half_rope if runtime_half_rope is not None else rope_dim // 2
)
return self.CompileKey(
**compile_key_fields,
heads_per_group=heads_per_group,
fp8_max=fp8_max,
quant_group_size=quant_group_size,
chunks_per_head=chunks_per_head,
rope_start=rope_start,
half_rope=half_rope,
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
hf_config = vllm_config.model_config.hf_config
if hf_config is None:
return []
tp_size = vllm_config.parallel_config.tensor_parallel_size
num_heads = int(getattr(hf_config, "num_attention_heads", 0) or 0)
num_groups = int(getattr(hf_config, "o_groups", 0) or 0)
if num_heads <= 0 or num_groups <= 0:
return []
local_heads = num_heads // tp_size
local_groups = num_groups // tp_size
if local_groups <= 0:
return []
capability = current_platform.get_device_capability()
if capability is None:
return []
head_dim = int(getattr(hf_config, "head_dim", 512) or 512)
rope_dim = int(getattr(hf_config, "qk_rope_head_dim", 64) or 64)
return self._trace_dispatch(self.dispatch)(
heads_per_group=local_heads // local_groups,
head_dim=head_dim,
nope_dim=head_dim - rope_dim,
rope_dim=rope_dim,
quant_group_size=128,
tma_aligned_scales=capability.major >= 10,
launch_pdl=(False, True),
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
head_dim = compile_key.chunks_per_head * compile_key.quant_group_size
fp8_dim = compile_key.heads_per_group * head_dim
scale_dtype = torch.int32 if compile_key.tma_aligned_scales else torch.float32
return dict(
o=TritonWarmupTensor(
torch.bfloat16,
shape=(1, compile_key.heads_per_group, head_dim),
),
positions=TritonWarmupTensor(torch.int64),
cos_sin_cache=TritonWarmupTensor(
torch.float32,
shape=(1, compile_key.half_rope * 2),
),
fp8_buf=TritonWarmupTensor(
torch.float8_e4m3fn,
shape=(compile_key.heads_per_group, 1, fp8_dim),
strides=(fp8_dim, fp8_dim, 1),
),
scale_buf=TritonWarmupTensor(
scale_dtype,
shape=(compile_key.heads_per_group, 1, 16),
strides=(16, 16, 16),
),
num_tokens=1,
heads_per_group=compile_key.heads_per_group,
quant_group_size=compile_key.quant_group_size,
chunks_per_head=compile_key.chunks_per_head,
rope_start=compile_key.rope_start,
half_rope=compile_key.half_rope,
tma_aligned_scales=compile_key.tma_aligned_scales,
fp8_max=compile_key.fp8_max,
launch_pdl=compile_key.launch_pdl,
grid=(1, compile_key.heads_per_group),
)
@kernel_launcher
def __call__(
self,
o: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
fp8_buf: torch.Tensor,
scale_buf: torch.Tensor,
num_tokens: int,
*,
heads_per_group: int,
quant_group_size: int,
chunks_per_head: int,
rope_start: int,
half_rope: int,
tma_aligned_scales: bool,
fp8_max: float,
launch_pdl: bool,
grid: tuple[int, int],
) -> LaunchSpec:
return grid, dict(
fp8_ptr=fp8_buf,
scale_ptr=scale_buf,
o_stride_token=o.stride(0),
o_stride_head=o.stride(1),
cache_stride_pos=cos_sin_cache.stride(0),
fp8_stride_group=fp8_buf.stride(0),
fp8_stride_token=fp8_buf.stride(1),
scale_stride_group=scale_buf.stride(0),
scale_stride_k=scale_buf.stride(2),
eps=1e-10,
QUANT_GROUP_SIZE=quant_group_size,
CHUNKS_PER_HEAD=chunks_per_head,
ROPE_START=rope_start,
HALF_ROPE=half_rope,
TMA_ALIGNED_SCALES=tma_aligned_scales,
launch_pdl=launch_pdl,
num_stages=1,
num_warps=1,
)
tl.store(scale_addrs, scales)
def fused_inv_rope_fp8_quant(
@@ -256,8 +438,8 @@ def _fused_inv_rope_fp8_quant_kernel_impl(
(scale_inner * tma_aligned_T, 1, tma_aligned_T),
)
grid = (tma_aligned_T, n_groups * heads_per_group)
use_gdc = current_platform.is_arch_support_pdl()
_fused_inv_rope_fp8_quant_per_head[grid](
launch_pdl = current_platform.is_arch_support_pdl()
_FUSED_INV_ROPE_FP8_QUANT_KERNEL(
o,
positions,
cos_sin_cache,
@@ -265,24 +447,14 @@ def _fused_inv_rope_fp8_quant_kernel_impl(
scale_buf,
num_tokens,
heads_per_group=heads_per_group,
o_stride_token=o.stride(0),
o_stride_head=o.stride(1),
cache_stride_pos=cos_sin_cache.stride(0),
fp8_stride_group=fp8_buf.stride(0),
fp8_stride_token=fp8_buf.stride(1),
scale_stride_group=scale_buf.stride(0),
scale_stride_k=scale_buf.stride(2),
quant_group_size=quant_group_size,
chunks_per_head=chunks_per_head,
rope_start=rope_start,
half_rope=half_rope,
tma_aligned_scales=tma_aligned_scales,
fp8_max=fp8_max,
eps=1e-10,
QUANT_GROUP_SIZE=quant_group_size,
CHUNKS_PER_HEAD=chunks_per_head,
ROPE_START=rope_start,
HALF_ROPE=half_rope,
TMA_ALIGNED_SCALES=tma_aligned_scales,
USE_GDC=use_gdc,
launch_pdl=use_gdc,
num_stages=1,
num_warps=1,
launch_pdl=launch_pdl,
grid=grid,
)
return fp8_buf, scale_buf
@@ -326,3 +498,6 @@ direct_register_custom_op(
op_func=_fused_inv_rope_fp8_quant_kernel_impl,
fake_impl=_fused_inv_rope_fp8_quant_kernel_fake,
)
_FUSED_INV_ROPE_FP8_QUANT_KERNEL = FusedInvRopeFP8QuantKernel()
@@ -17,9 +17,19 @@ A single grid (T, hc_mult+1) drives both norms: task 0 is enorm on
inputs_embeds[token, :], task k+1 is hnorm on previous_hidden_states[token, k, :].
"""
from dataclasses import dataclass
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
kernel_launcher,
)
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import next_power_of_2
@triton.jit
@@ -40,164 +50,291 @@ def _rmsnorm_row(
tl.store(out_row_ptr + block, y.to(out_row_ptr.dtype.element_ty), mask=mask)
@triton.jit
def _fused_mtp_input_rmsnorm_kernel(
inputs_embeds_ptr,
positions_ptr,
prev_hidden_ptr,
enorm_weight_ptr,
hnorm_weight_ptr,
enorm_out_ptr,
hnorm_out_ptr,
eps,
HIDDEN: tl.constexpr,
HC_MULT: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
class FusedMTPInputRMSNormKernel(
VllmTritonJitKernel["FusedMTPInputRMSNormKernel.CompileKey"]
):
# int64 token index so per-token offsets don't overflow int32 at
# large num_tokens (matches the convention in fused_q_kv_rmsnorm).
token_idx = tl.program_id(0).to(tl.int64)
pid_task = tl.program_id(1)
@dataclass(frozen=True)
class CompileKey:
dtype: torch.dtype
hidden: int
hc_mult: int
block_size: int
eps: float
block = tl.arange(0, BLOCK_SIZE)
mask = block < HIDDEN
@staticmethod
@triton.jit
def kernel(
inputs_embeds_ptr,
positions_ptr,
prev_hidden_ptr,
enorm_weight_ptr,
hnorm_weight_ptr,
enorm_out_ptr,
hnorm_out_ptr,
eps,
HIDDEN: tl.constexpr,
HC_MULT: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
# int64 token index so per-token offsets don't overflow int32 at
# large num_tokens (matches the convention in fused_q_kv_rmsnorm).
token_idx = tl.program_id(0).to(tl.int64)
pid_task = tl.program_id(1)
if pid_task == 0:
# enorm path: load inputs_embeds[token, :] then zero-mask at pos==0.
# Math is preserved: pos==0 → x=0 → variance=0 → RMSNorm output is 0
# regardless of weight, matching torch.where(pos==0, 0, x) + RMSNorm.
pos = tl.load(positions_ptr + token_idx)
keep = pos != 0
x = tl.load(
inputs_embeds_ptr + token_idx * HIDDEN + block, mask=mask, other=0.0
block = tl.arange(0, BLOCK_SIZE)
mask = block < HIDDEN
if pid_task == 0:
# enorm path: load inputs_embeds[token, :] then zero-mask at pos==0.
# Math is preserved: pos==0 → x=0 → variance=0 → RMSNorm output is 0
# regardless of weight, matching torch.where(pos==0, 0, x) + RMSNorm.
pos = tl.load(positions_ptr + token_idx)
keep = pos != 0
x = tl.load(
inputs_embeds_ptr + token_idx * HIDDEN + block, mask=mask, other=0.0
)
x = tl.where(keep, x, 0.0)
_rmsnorm_row(
x,
enorm_weight_ptr,
enorm_out_ptr + token_idx * HIDDEN,
block,
mask,
eps,
HIDDEN,
)
else:
# hnorm path: load prev_hidden[token, slot, :].
slot = pid_task - 1
row_offset = (token_idx * HC_MULT + slot) * HIDDEN
x = tl.load(prev_hidden_ptr + row_offset + block, mask=mask, other=0.0)
_rmsnorm_row(
x,
hnorm_weight_ptr,
hnorm_out_ptr + row_offset,
block,
mask,
eps,
HIDDEN,
)
def dispatch( # type: ignore[override]
self,
*,
dtype: torch.dtype,
hidden: int,
hc_mult: int,
eps: float,
) -> CompileKey:
return self.CompileKey(
dtype=dtype,
hidden=hidden,
hc_mult=hc_mult,
block_size=next_power_of_2(hidden),
eps=eps,
)
x = tl.where(keep, x, 0.0)
_rmsnorm_row(
x,
enorm_weight_ptr,
enorm_out_ptr + token_idx * HIDDEN,
block,
mask,
eps,
HIDDEN,
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
hf_config = vllm_config.model_config.hf_config
if int(getattr(hf_config, "num_nextn_predict_layers", 0) or 0) <= 0:
return []
return self._trace_dispatch(self.dispatch)(
dtype=vllm_config.model_config.dtype,
hidden=int(hf_config.hidden_size),
hc_mult=int(hf_config.hc_mult),
eps=float(hf_config.rms_norm_eps),
)
else:
# hnorm path: load prev_hidden[token, slot, :].
slot = pid_task - 1
row_offset = (token_idx * HC_MULT + slot) * HIDDEN
x = tl.load(prev_hidden_ptr + row_offset + block, mask=mask, other=0.0)
_rmsnorm_row(
x,
hnorm_weight_ptr,
hnorm_out_ptr + row_offset,
block,
mask,
eps,
HIDDEN,
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
return dict(
inputs_embeds=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.hidden),
),
positions=TritonWarmupTensor(torch.int64),
previous_hidden_states=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.hc_mult, compile_key.hidden),
),
enorm_weight=TritonWarmupTensor(
compile_key.dtype,
shape=(compile_key.hidden,),
),
hnorm_weight=TritonWarmupTensor(
compile_key.dtype,
shape=(compile_key.hidden,),
),
_outputs=(
TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.hidden),
),
TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.hc_mult, compile_key.hidden),
),
),
eps=compile_key.eps,
hc_mult=compile_key.hc_mult,
)
@kernel_launcher
def __call__(
self,
inputs_embeds: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
enorm_weight: torch.Tensor,
hnorm_weight: torch.Tensor,
eps: float,
hc_mult: int,
*,
_outputs: tuple[Any, Any] | None = None,
) -> LaunchSpec:
num_tokens, hidden = inputs_embeds.shape
if _outputs is None:
assert inputs_embeds.ndim == 2
assert previous_hidden_states.ndim == 3
assert previous_hidden_states.shape[1] == hc_mult
assert inputs_embeds.shape[0] == previous_hidden_states.shape[0], (
"token dim mismatch"
)
assert (
hidden
== previous_hidden_states.shape[2]
== enorm_weight.shape[0]
== hnorm_weight.shape[0]
)
assert (
inputs_embeds.is_contiguous() and previous_hidden_states.is_contiguous()
)
assert enorm_weight.is_contiguous() and hnorm_weight.is_contiguous()
_outputs = (
torch.empty_like(inputs_embeds),
torch.empty_like(previous_hidden_states),
)
enorm_out, hnorm_out = _outputs
grid = None if num_tokens == 0 else (num_tokens, hc_mult + 1)
return (
grid,
dict(
prev_hidden_ptr=previous_hidden_states,
enorm_out_ptr=enorm_out,
hnorm_out_ptr=hnorm_out,
HIDDEN=hidden,
HC_MULT=hc_mult,
BLOCK_SIZE=next_power_of_2(hidden),
),
_outputs,
)
@triton.jit
def _mtp_shared_head_rmsnorm_kernel(
x_ptr,
weight_ptr,
out_ptr,
eps,
HIDDEN: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
class MTPSharedHeadRMSNormKernel(
VllmTritonJitKernel["MTPSharedHeadRMSNormKernel.CompileKey"]
):
token_idx = tl.program_id(0).to(tl.int64)
block = tl.arange(0, BLOCK_SIZE)
mask = block < HIDDEN
x = tl.load(x_ptr + token_idx * HIDDEN + block, mask=mask, other=0.0)
_rmsnorm_row(
x,
@dataclass(frozen=True)
class CompileKey:
dtype: torch.dtype
hidden: int
block_size: int
eps: float
@staticmethod
@triton.jit
def kernel(
x_ptr,
weight_ptr,
out_ptr + token_idx * HIDDEN,
block,
mask,
out_ptr,
eps,
HIDDEN,
)
HIDDEN: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
token_idx = tl.program_id(0).to(tl.int64)
block = tl.arange(0, BLOCK_SIZE)
mask = block < HIDDEN
x = tl.load(x_ptr + token_idx * HIDDEN + block, mask=mask, other=0.0)
_rmsnorm_row(
x,
weight_ptr,
out_ptr + token_idx * HIDDEN,
block,
mask,
eps,
HIDDEN,
)
def dispatch( # type: ignore[override]
self,
*,
dtype: torch.dtype,
hidden: int,
eps: float,
) -> CompileKey:
return self.CompileKey(
dtype=dtype,
hidden=hidden,
block_size=next_power_of_2(hidden),
eps=eps,
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
hf_config = vllm_config.model_config.hf_config
if int(getattr(hf_config, "num_nextn_predict_layers", 0) or 0) <= 0:
return []
return self._trace_dispatch(self.dispatch)(
dtype=vllm_config.model_config.dtype,
hidden=int(hf_config.hidden_size),
eps=float(hf_config.rms_norm_eps),
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
return dict(
hidden_states=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.hidden),
),
weight=TritonWarmupTensor(
compile_key.dtype,
shape=(compile_key.hidden,),
),
_output=TritonWarmupTensor(
compile_key.dtype,
shape=(1, compile_key.hidden),
),
eps=compile_key.eps,
)
@kernel_launcher
def __call__(
self,
hidden_states: torch.Tensor,
weight: torch.Tensor,
eps: float,
*,
_output: Any | None = None,
) -> LaunchSpec:
num_tokens, hidden = hidden_states.shape
if _output is None:
assert hidden_states.ndim == 2
assert hidden_states.is_contiguous()
assert weight.is_contiguous()
_output = torch.empty_like(hidden_states)
grid = None if num_tokens == 0 else (num_tokens,)
return (
grid,
dict(
x_ptr=hidden_states,
out_ptr=_output,
HIDDEN=hidden,
BLOCK_SIZE=next_power_of_2(hidden),
),
_output,
)
def mtp_shared_head_rmsnorm(
hidden_states: torch.Tensor,
weight: torch.Tensor,
eps: float,
) -> torch.Tensor:
"""RMSNorm for MTP's SharedHead.norm, on (T, H) bf16 input.
Uses the same ``_rmsnorm_row`` body as ``fused_mtp_input_rmsnorm`` so the
MTP draft path runs one consistent RMSNorm implementation end to end.
"""
assert hidden_states.ndim == 2
assert hidden_states.is_contiguous()
assert weight.is_contiguous()
num_tokens, hidden = hidden_states.shape
out = torch.empty_like(hidden_states)
if num_tokens == 0:
return out
block_size = triton.next_power_of_2(hidden)
_mtp_shared_head_rmsnorm_kernel[(num_tokens,)](
hidden_states,
weight,
out,
eps,
HIDDEN=hidden,
BLOCK_SIZE=block_size,
)
return out
def fused_mtp_input_rmsnorm(
inputs_embeds: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
enorm_weight: torch.Tensor,
hnorm_weight: torch.Tensor,
eps: float,
hc_mult: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Returns (enorm_out, hnorm_out).
enorm_out has the same shape as inputs_embeds (2D, [T, H]).
hnorm_out has the same shape as previous_hidden_states (3D, [T, hc_mult, H]).
previous_hidden_states must already be reshaped to 3D.
"""
assert inputs_embeds.ndim == 2
assert previous_hidden_states.ndim == 3
assert previous_hidden_states.shape[1] == hc_mult
assert inputs_embeds.shape[0] == previous_hidden_states.shape[0], (
"token dim mismatch"
)
assert (
inputs_embeds.shape[1]
== previous_hidden_states.shape[2]
== enorm_weight.shape[0]
== hnorm_weight.shape[0]
)
assert inputs_embeds.is_contiguous() and previous_hidden_states.is_contiguous()
assert enorm_weight.is_contiguous() and hnorm_weight.is_contiguous()
num_tokens, hidden = inputs_embeds.shape
enorm_out = torch.empty_like(inputs_embeds)
hnorm_out = torch.empty_like(previous_hidden_states)
if num_tokens == 0:
return enorm_out, hnorm_out
block_size = triton.next_power_of_2(hidden)
_fused_mtp_input_rmsnorm_kernel[(num_tokens, hc_mult + 1)](
inputs_embeds,
positions,
previous_hidden_states,
enorm_weight,
hnorm_weight,
enorm_out,
hnorm_out,
eps,
HIDDEN=hidden,
HC_MULT=hc_mult,
BLOCK_SIZE=block_size,
)
return enorm_out, hnorm_out
_FUSED_MTP_INPUT_RMSNORM_KERNEL = FusedMTPInputRMSNormKernel()
_MTP_SHARED_HEAD_RMSNORM_KERNEL = MTPSharedHeadRMSNormKernel()
@@ -1,101 +1,193 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
kernel_launcher,
)
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import next_power_of_2
def save_partial_states(
kv: torch.Tensor,
score: torch.Tensor,
ape: torch.Tensor,
positions: torch.Tensor,
state_cache: torch.Tensor,
slot_mapping: torch.Tensor,
block_size: int,
state_width: int,
compress_ratio: int,
pdl_kwargs: dict | None = None,
) -> None:
"""Write packed [kv, score+ape] partial states into the compressor cache.
One program per token; pads (slot_id == -1) are skipped.
"""
num_actual = slot_mapping.shape[0]
head_size = kv.shape[-1]
_save_partial_states_kernel[(num_actual,)](
kv,
kv.stride(0),
score,
score.stride(0),
ape,
ape.stride(0),
positions,
state_cache,
state_cache.stride(0),
state_cache.stride(1),
slot_mapping,
block_size,
HEAD_SIZE=head_size,
TRITON_BLOCK_SIZE=triton.next_power_of_2(head_size),
STATE_WIDTH=state_width,
COMPRESS_RATIO=compress_ratio,
**(pdl_kwargs or {}),
)
@triton.jit
def _save_partial_states_kernel(
kv_ptr,
kv_stride,
score_ptr,
score_stride,
ape_ptr,
ape_stride,
positions_ptr,
state_cache_ptr,
state_cache_stride0,
state_cache_stride1,
slot_mapping_ptr,
block_size,
HEAD_SIZE: tl.constexpr,
TRITON_BLOCK_SIZE: tl.constexpr,
# state_cache last dim packs [kv_state, score_state], each STATE_WIDTH wide.
STATE_WIDTH: tl.constexpr,
COMPRESS_RATIO: tl.constexpr,
class SavePartialStatesKernel(
VllmTritonJitKernel["SavePartialStatesKernel.CompileKey"]
):
token_idx = tl.program_id(0)
slot_id = tl.load(slot_mapping_ptr + token_idx)
@dataclass(frozen=True)
class CompileKey:
head_size: int
triton_block_size: int
state_width: int
compress_ratio: int
kv_stride: int
score_stride: int
ape_stride: int
state_cache_stride0: int
state_cache_stride1: int
block_size: int
# Skip padded / invalid tokens (slot_id == -1 is the PAD sentinel used
# by vLLM). During CUDA graph replay the batch may contain padding
# tokens whose slot_mapping is -1; writing to kv_state[-1] would be an
# illegal memory access.
if slot_id < 0:
return
@staticmethod
@triton.jit
def kernel(
kv_ptr,
kv_stride,
score_ptr,
score_stride,
ape_ptr,
ape_stride,
positions_ptr,
state_cache_ptr,
state_cache_stride0,
state_cache_stride1,
slot_mapping_ptr,
block_size,
HEAD_SIZE: tl.constexpr,
TRITON_BLOCK_SIZE: tl.constexpr,
# state_cache last dim packs [kv_state, score_state], each STATE_WIDTH wide.
STATE_WIDTH: tl.constexpr,
COMPRESS_RATIO: tl.constexpr,
):
token_idx = tl.program_id(0)
slot_id = tl.load(slot_mapping_ptr + token_idx)
block_idx = slot_id // block_size
pos_in_block = slot_id % block_size
base_ptr = (
state_cache_ptr
+ block_idx * state_cache_stride0
+ pos_in_block * state_cache_stride1
)
# Skip padded / invalid tokens (slot_id == -1 is the PAD sentinel used
# by vLLM). During CUDA graph replay the batch may contain padding
# tokens whose slot_mapping is -1; writing to kv_state[-1] would be an
# illegal memory access.
if slot_id < 0:
return
block = tl.arange(0, TRITON_BLOCK_SIZE)
mask = block < HEAD_SIZE
block_idx = slot_id // block_size
pos_in_block = slot_id % block_size
base_ptr = (
state_cache_ptr
+ block_idx * state_cache_stride0
+ pos_in_block * state_cache_stride1
)
kv = tl.load(kv_ptr + token_idx * kv_stride + block, mask=mask)
tl.store(base_ptr + block, kv, mask=mask)
block = tl.arange(0, TRITON_BLOCK_SIZE)
mask = block < HEAD_SIZE
# Fused: score += ape[position % compress_ratio]
position = tl.load(positions_ptr + token_idx)
ape_row = position % COMPRESS_RATIO
ape = tl.load(ape_ptr + ape_row * ape_stride + block, mask=mask)
score = tl.load(score_ptr + token_idx * score_stride + block, mask=mask)
tl.store(
base_ptr + STATE_WIDTH + block,
score + ape,
mask=mask,
)
kv = tl.load(kv_ptr + token_idx * kv_stride + block, mask=mask)
tl.store(base_ptr + block, kv, mask=mask)
# Fused: score += ape[position % compress_ratio]
position = tl.load(positions_ptr + token_idx)
ape_row = position % COMPRESS_RATIO
ape = tl.load(ape_ptr + ape_row * ape_stride + block, mask=mask)
score = tl.load(score_ptr + token_idx * score_stride + block, mask=mask)
tl.store(
base_ptr + STATE_WIDTH + block,
score + ape,
mask=mask,
)
def dispatch( # type: ignore[override]
self,
*,
head_size: int,
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(
**compile_key_fields,
head_size=head_size,
triton_block_size=next_power_of_2(head_size),
)
def get_warmup_keys(
self,
*,
head_dim: int,
compress_ratio: int,
) -> list[CompileKey]:
if head_dim <= 0 or compress_ratio not in (4, 128):
return []
coefficient = 2 if compress_ratio == 4 else 1
return self._trace_dispatch(self.dispatch)(
head_size=coefficient * head_dim,
state_width=coefficient * head_dim,
compress_ratio=compress_ratio,
kv_stride=2 * coefficient * head_dim,
score_stride=2 * coefficient * head_dim,
ape_stride=coefficient * head_dim,
state_cache_stride0=8 * coefficient * head_dim,
state_cache_stride1=2 * coefficient * head_dim,
block_size=4 if compress_ratio == 4 else 8,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
return dict(
kv=TritonWarmupTensor(
torch.float32,
shape=(1, compile_key.head_size),
strides=(compile_key.kv_stride, 1),
),
score=TritonWarmupTensor(
torch.float32,
shape=(1, 1),
strides=(compile_key.score_stride, 1),
),
ape=TritonWarmupTensor(
torch.float32,
shape=(1, 1),
strides=(compile_key.ape_stride, 1),
),
positions=TritonWarmupTensor(torch.int64),
state_cache=TritonWarmupTensor(
torch.float32,
shape=(1, 1, compile_key.state_width),
strides=(
compile_key.state_cache_stride0,
compile_key.state_cache_stride1,
1,
),
),
slot_mapping=TritonWarmupTensor(torch.int64),
block_size=compile_key.block_size,
state_width=compile_key.state_width,
compress_ratio=compile_key.compress_ratio,
)
@kernel_launcher
def __call__(
self,
kv: torch.Tensor,
score: torch.Tensor,
ape: torch.Tensor,
positions: torch.Tensor,
state_cache: torch.Tensor,
slot_mapping: torch.Tensor,
block_size: int,
state_width: int,
compress_ratio: int,
pdl_kwargs: dict | None = None,
) -> LaunchSpec:
"""Write packed [kv, score+ape] partial states into the compressor cache.
One program per token; pads (slot_id == -1) are skipped.
"""
num_actual = slot_mapping.shape[0]
head_size = kv.shape[-1]
return (num_actual,), dict(
kv_stride=kv.stride(0),
score_stride=score.stride(0),
ape_stride=ape.stride(0),
state_cache_stride0=state_cache.stride(0),
state_cache_stride1=state_cache.stride(1),
block_size=block_size,
HEAD_SIZE=head_size,
TRITON_BLOCK_SIZE=next_power_of_2(head_size),
STATE_WIDTH=state_width,
COMPRESS_RATIO=compress_ratio,
**(pdl_kwargs or {}),
)
_SAVE_PARTIAL_STATES_KERNEL = SavePartialStatesKernel()
+14 -2
View File
@@ -18,7 +18,7 @@ from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import (
)
from vllm.models.deepseek_v4.common.ops.fused_indexer_q import MXFP4_BLOCK_SIZE
from vllm.models.deepseek_v4.common.ops.save_partial_states import (
save_partial_states,
_SAVE_PARTIAL_STATES_KERNEL,
)
from vllm.platforms import current_platform
from vllm.v1.attention.backend import (
@@ -306,6 +306,18 @@ class DeepseekCompressor(nn.Module):
f"Unsupported head_dim for fused quant+cache: {self.head_dim}"
)
if vllm_config.kernel_config.enable_jit_warmup:
_SAVE_PARTIAL_STATES_KERNEL.register_warmup(
head_dim=self.head_dim,
compress_ratio=self.compress_ratio,
)
if self.head_dim != 512:
from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import ( # noqa: E501
_FUSED_KV_COMPRESS_NORM_ROPE_INSERT_INDEXER_TRITON_KERNEL,
)
_FUSED_KV_COMPRESS_NORM_ROPE_INSERT_INDEXER_TRITON_KERNEL.register_warmup()
def forward(
self,
# [num_tokens, 2 * self.coff * self.head_dim]
@@ -351,7 +363,7 @@ class DeepseekCompressor(nn.Module):
# GEMM; state_cache from this kernel) but neither emits/waits on PDL
# grid dependency primitives, so launch_pdl=True caused a
# read-after-write race and non-deterministic output.
save_partial_states(
_SAVE_PARTIAL_STATES_KERNEL(
kv=kv,
score=score,
ape=self.ape,
+9 -5
View File
@@ -51,9 +51,9 @@ from vllm.models.common.ops.sequence_parallel import (
sp_padding_mask,
sp_shard,
)
from vllm.models.deepseek_v4.common.ops import (
fused_mtp_input_rmsnorm,
mtp_shared_head_rmsnorm,
from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import (
_FUSED_MTP_INPUT_RMSNORM_KERNEL,
_MTP_SHARED_HEAD_RMSNORM_KERNEL,
)
from vllm.sequence import IntermediateTensors
@@ -141,6 +141,10 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
aux_stream_list=aux_stream_list,
)
if vllm_config.kernel_config.enable_jit_warmup:
_FUSED_MTP_INPUT_RMSNORM_KERNEL.register_warmup()
_MTP_SHARED_HEAD_RMSNORM_KERNEL.register_warmup()
def forward(
self,
input_ids: torch.Tensor,
@@ -157,7 +161,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
-1, self.hc_mult, self.config.hidden_size
)
# Fused: mask inputs at position 0 (not needed by MTP), enorm, hnorm.
inputs_embeds, previous_hidden_states = fused_mtp_input_rmsnorm(
inputs_embeds, previous_hidden_states = _FUSED_MTP_INPUT_RMSNORM_KERNEL(
inputs_embeds,
positions,
previous_hidden_states,
@@ -270,7 +274,7 @@ class DeepSeekV4MultiTokenPredictor(nn.Module):
mtp_layer.rms_norm_eps,
mtp_layer.hc_eps,
)
hidden_states = mtp_shared_head_rmsnorm(
hidden_states = _MTP_SHARED_HEAD_RMSNORM_KERNEL(
hidden_states,
mtp_layer.shared_head.norm.weight.data,
mtp_layer.shared_head.norm.variance_epsilon,
+161 -65
View File
@@ -9,6 +9,12 @@ import torch
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
kernel_launcher,
)
from vllm.platforms import current_platform
from vllm.platforms.interface import DeviceCapability
from vllm.triton_utils import tl, triton
@@ -338,90 +344,180 @@ def build_c128a_topk_metadata(
if num_tokens == 0:
return global_decode, decode_lens, prefill_local
_build_c128a_topk_metadata_kernel[(num_tokens,)](
_BUILD_C128A_TOPK_METADATA_KERNEL(
global_decode_buffer,
global_decode_buffer.stride(0),
decode_lens_buffer,
prefill_buffer,
prefill_buffer.stride(0),
positions,
compress_ratio,
max_compressed_tokens,
num_decode_tokens,
token_to_req_indices,
block_table,
block_table.stride(0),
block_size,
slot_mapping,
BLOCK_SIZE=1024,
)
return global_decode, decode_lens, prefill_local
@triton.jit
def _build_c128a_topk_metadata_kernel(
# Decode outputs
global_decode_ptr,
global_decode_stride,
decode_lens_ptr,
# Prefill output
prefill_local_ptr,
prefill_local_stride,
# Inputs
positions_ptr,
compress_ratio,
max_compressed_tokens,
num_decode_tokens,
token_to_req_indices_ptr,
block_table_ptr,
block_table_stride,
block_size,
slot_mapping_ptr,
BLOCK_SIZE: tl.constexpr,
class BuildC128ATopkMetadataKernel(
VllmTritonJitKernel["BuildC128ATopkMetadataKernel.CompileKey"]
):
token_idx = tl.program_id(0)
position = tl.load(positions_ptr + token_idx)
num_compressed = (position + 1) // compress_ratio
num_compressed = tl.minimum(num_compressed, max_compressed_tokens)
is_decode = token_idx < num_decode_tokens
@dataclass(frozen=True)
class CompileKey:
compress_ratio: int
max_compressed_tokens: int
block_size: int
triton_block_size: int
if is_decode:
# --- Decode: block-table lookup → global slot ids + count ---
is_valid_token = tl.load(slot_mapping_ptr + token_idx) >= 0
req_idx = tl.load(token_to_req_indices_ptr + token_idx)
count = tl.zeros((), dtype=tl.int32)
for i in range(0, max_compressed_tokens, BLOCK_SIZE):
offset = i + tl.arange(0, BLOCK_SIZE)
mask = offset < max_compressed_tokens
is_valid = offset < num_compressed
@staticmethod
@triton.jit(
do_not_specialize=[
"global_decode_stride",
"prefill_local_stride",
"num_decode_tokens",
"block_table_stride",
]
)
def kernel(
# Decode outputs
global_decode_ptr,
global_decode_stride,
decode_lens_ptr,
# Prefill output
prefill_local_ptr,
prefill_local_stride,
# Inputs
positions_ptr,
compress_ratio,
max_compressed_tokens,
num_decode_tokens,
token_to_req_indices_ptr,
block_table_ptr,
block_table_stride,
block_size,
slot_mapping_ptr,
BLOCK_SIZE: tl.constexpr,
):
token_idx = tl.program_id(0)
position = tl.load(positions_ptr + token_idx)
num_compressed = (position + 1) // compress_ratio
num_compressed = tl.minimum(num_compressed, max_compressed_tokens)
is_decode = token_idx < num_decode_tokens
if is_decode:
# --- Decode: block-table lookup → global slot ids + count ---
is_valid_token = tl.load(slot_mapping_ptr + token_idx) >= 0
req_idx = tl.load(token_to_req_indices_ptr + token_idx)
count = tl.zeros((), dtype=tl.int32)
for i in range(0, max_compressed_tokens, BLOCK_SIZE):
offset = i + tl.arange(0, BLOCK_SIZE)
mask = offset < max_compressed_tokens
is_valid = offset < num_compressed
block_indices = offset // block_size
block_numbers = tl.load(
block_table_ptr + req_idx * block_table_stride + block_indices,
mask=mask & is_valid,
)
block_offsets = offset % block_size
slot_ids = block_numbers * block_size + block_offsets
slot_ids = tl.where(is_valid, slot_ids, -1)
tl.store(
global_decode_ptr + token_idx * global_decode_stride + offset,
slot_ids,
mask=mask,
)
count += tl.sum(is_valid.to(tl.int32), axis=0)
block_indices = offset // block_size
block_numbers = tl.load(
block_table_ptr + req_idx * block_table_stride + block_indices,
mask=mask & is_valid,
)
block_offsets = offset % block_size
slot_ids = block_numbers * block_size + block_offsets
slot_ids = tl.where(is_valid, slot_ids, -1)
tl.store(
global_decode_ptr + token_idx * global_decode_stride + offset,
slot_ids,
mask=mask,
decode_lens_ptr + token_idx,
tl.where(is_valid_token, count, 0),
)
count += tl.sum(is_valid.to(tl.int32), axis=0)
else:
# --- Prefill: write local indices ---
pfx_idx = token_idx - num_decode_tokens
for i in range(0, max_compressed_tokens, BLOCK_SIZE):
offset = i + tl.arange(0, BLOCK_SIZE)
mask = offset < max_compressed_tokens
tl.store(
prefill_local_ptr + pfx_idx * prefill_local_stride + offset,
tl.where(offset < num_compressed, offset, -1),
mask=mask,
)
tl.store(
decode_lens_ptr + token_idx,
tl.where(is_valid_token, count, 0),
def dispatch( # type: ignore[override]
self,
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(**compile_key_fields)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
hf_config = vllm_config.model_config.hf_config
if getattr(hf_config, "model_type", None) != "deepseek_v4":
return []
compress_ratio = 128
max_model_len = vllm_config.model_config.max_model_len
if max_model_len <= 0:
return []
max_compressed_tokens = cdiv(max_model_len, compress_ratio)
max_compressed_tokens = (
cdiv(max_compressed_tokens, _C128A_TOPK_ALIGNMENT) * _C128A_TOPK_ALIGNMENT
)
else:
# --- Prefill: write local indices ---
pfx_idx = token_idx - num_decode_tokens
for i in range(0, max_compressed_tokens, BLOCK_SIZE):
offset = i + tl.arange(0, BLOCK_SIZE)
mask = offset < max_compressed_tokens
tl.store(
prefill_local_ptr + pfx_idx * prefill_local_stride + offset,
tl.where(offset < num_compressed, offset, -1),
mask=mask,
)
return self._trace_dispatch(self.dispatch)(
compress_ratio=compress_ratio,
max_compressed_tokens=max_compressed_tokens,
# DeepSeek V4 sparse MLA uses 256-token KV pages; C128A metadata
# works in compressed-token units.
block_size=256 // compress_ratio,
triton_block_size=1024,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
int32_ptr = TritonWarmupTensor(torch.int32)
return dict(
global_decode_buffer=TritonWarmupTensor(
torch.int32, shape=(1, 1), strides=(1, 1)
),
decode_lens_buffer=int32_ptr,
prefill_buffer=TritonWarmupTensor(
torch.int32, shape=(1, 1), strides=(1, 1)
),
positions=TritonWarmupTensor(torch.int64),
compress_ratio=compile_key.compress_ratio,
max_compressed_tokens=compile_key.max_compressed_tokens,
num_decode_tokens=0,
token_to_req_indices=int32_ptr,
block_table=TritonWarmupTensor(torch.int32, shape=(1, 1), strides=(1, 1)),
block_size=compile_key.block_size,
slot_mapping=TritonWarmupTensor(torch.int64),
)
@kernel_launcher
def __call__(
self,
global_decode_buffer: torch.Tensor,
decode_lens_buffer: torch.Tensor,
prefill_buffer: torch.Tensor,
positions: torch.Tensor,
compress_ratio: int,
max_compressed_tokens: int,
num_decode_tokens: int,
token_to_req_indices: torch.Tensor,
block_table: torch.Tensor,
block_size: int,
slot_mapping: torch.Tensor,
) -> LaunchSpec:
return (positions.shape[0],), dict(
global_decode_ptr=global_decode_buffer,
global_decode_stride=global_decode_buffer.stride(0),
decode_lens_ptr=decode_lens_buffer,
prefill_local_ptr=prefill_buffer,
prefill_local_stride=prefill_buffer.stride(0),
block_table_stride=block_table.stride(0),
BLOCK_SIZE=1024,
)
_BUILD_C128A_TOPK_METADATA_KERNEL = BuildC128ATopkMetadataKernel()
+9 -5
View File
@@ -39,9 +39,9 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.deepseek_mtp import SharedHead
from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weight_name
from vllm.model_executor.models.utils import maybe_prefix
from vllm.models.deepseek_v4.common.ops import (
fused_mtp_input_rmsnorm,
mtp_shared_head_rmsnorm,
from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import (
_FUSED_MTP_INPUT_RMSNORM_KERNEL,
_MTP_SHARED_HEAD_RMSNORM_KERNEL,
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
@@ -130,6 +130,10 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
self.hc_head_op = HCHeadOp()
if vllm_config.kernel_config.enable_jit_warmup:
_FUSED_MTP_INPUT_RMSNORM_KERNEL.register_warmup()
_MTP_SHARED_HEAD_RMSNORM_KERNEL.register_warmup()
def forward(
self,
input_ids: torch.Tensor,
@@ -146,7 +150,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
-1, self.hc_mult, self.config.hidden_size
)
# Fused: mask inputs at position 0 (not needed by MTP), enorm, hnorm.
inputs_embeds, previous_hidden_states = fused_mtp_input_rmsnorm(
inputs_embeds, previous_hidden_states = _FUSED_MTP_INPUT_RMSNORM_KERNEL(
inputs_embeds,
positions,
previous_hidden_states,
@@ -253,7 +257,7 @@ class DeepSeekV4MultiTokenPredictor(nn.Module):
mtp_layer.rms_norm_eps,
mtp_layer.hc_eps,
)
hidden_states = mtp_shared_head_rmsnorm(
hidden_states = _MTP_SHARED_HEAD_RMSNORM_KERNEL(
hidden_states,
mtp_layer.shared_head.norm.weight.data,
mtp_layer.shared_head.norm.variance_epsilon,