diff --git a/tests/kernels/core/test_fused_q_kv_rmsnorm.py b/tests/kernels/core/test_fused_q_kv_rmsnorm.py index c10590085e..8debe7c3a3 100644 --- a/tests/kernels/core/test_fused_q_kv_rmsnorm.py +++ b/tests/kernels/core/test_fused_q_kv_rmsnorm.py @@ -13,7 +13,9 @@ from __future__ import annotations import pytest import torch -from vllm.models.common.ops import fused_q_kv_rmsnorm +from vllm.models.common.ops.fused_qk_rmsnorm import ( + fused_q_kv_rmsnorm, +) from vllm.platforms import current_platform pytestmark = pytest.mark.skipif( diff --git a/tests/model_executor/test_jit_warmup.py b/tests/model_executor/test_jit_warmup.py index b3bbe3231f..62ecb05f6b 100644 --- a/tests/model_executor/test_jit_warmup.py +++ b/tests/model_executor/test_jit_warmup.py @@ -431,6 +431,40 @@ def test_dispatch_can_forward_compile_key_fields() -> None: kernel.compile_key({"tokens": 5, "mode": 2, "extra": 1}) +def test_forwarded_compile_key_fields_exclude_predicate_only_inputs() -> None: + class ForwardingKernel(VllmJitKernel["ForwardingKernel.CompileKey"]): + @dataclass(frozen=True) + class CompileKey: + mode: int + block_size: int + + def dispatch( # type: ignore[override] + self, + *, + tokens: int, + **compile_key_fields: int, + ) -> CompileKey: + return self.CompileKey( + **compile_key_fields, + block_size=_round_up(tokens, multiple=4), + ) + + def get_warmup_keys(self) -> list[CompileKey]: + return self._trace_dispatch(self.dispatch)( + tokens=(1, 5), + mode=2, + max_tokens=4, + _when=lambda *, tokens, max_tokens: tokens <= max_tokens, + ) + + def compile(self, compile_key: CompileKey) -> None: + pass + + assert ForwardingKernel().get_warmup_keys() == [ + ForwardingKernel.CompileKey(mode=2, block_size=4) + ] + + def test_dispatch_supports_tuple_and_mapping_subscriptions() -> None: class SubscriptKernel(VllmJitKernel["SubscriptKernel.CompileKey"]): @dataclass(frozen=True) diff --git a/vllm/model_executor/warmup/jit_warmup.py b/vllm/model_executor/warmup/jit_warmup.py index d3216da111..0be4bddea6 100644 --- a/vllm/model_executor/warmup/jit_warmup.py +++ b/vllm/model_executor/warmup/jit_warmup.py @@ -14,8 +14,8 @@ from abc import ABC, abstractmethod from collections.abc import Callable, Iterable, Iterator, Mapping from contextlib import contextmanager from contextvars import ContextVar -from dataclasses import dataclass -from typing import Any, Generic, TypeVar +from dataclasses import dataclass, fields +from typing import Any, Generic, TypeVar, cast __all__ = [ "JitWarmupRegistry", @@ -597,6 +597,16 @@ class VllmJitKernel(Generic[CompileKeyT], ABC): predicate_trace = ( _trace_warmup_predicate(_when) if _when is not None else None ) + predicate_only_names: frozenset[str] = frozenset() + if predicate_trace is not None: + compile_key_fields = frozenset( + field.name for field in fields(cast(Any, self.CompileKey)) + ) + predicate_only_names = ( + predicate_trace.input_names + - compile_key_dispatch_trace.input_names + - compile_key_fields + ) # Unmatched **kwargs fields also belong to the expansion space. available_names = set(kwargs).union( *(group.rows[0] for group in input_groups) @@ -638,7 +648,12 @@ class VllmJitKernel(Generic[CompileKeyT], ABC): ): continue compile_key = compile_key_dispatch_trace.compile_key( - self.CompileKey, dispatch_values + self.CompileKey, + { + name: value + for name, value in dispatch_values.items() + if name not in predicate_only_names + }, ) compile_keys[compile_key] = None return list(compile_keys) diff --git a/vllm/model_executor/warmup/jit_warmup_triton_helper.py b/vllm/model_executor/warmup/jit_warmup_triton_helper.py index b9b991ef82..7fffa12ec9 100644 --- a/vllm/model_executor/warmup/jit_warmup_triton_helper.py +++ b/vllm/model_executor/warmup/jit_warmup_triton_helper.py @@ -15,7 +15,13 @@ from vllm.model_executor.warmup.jit_warmup import ( ) CompileKeyT = TypeVar("CompileKeyT") -LaunchSpec = tuple[tuple[int, ...], dict[str, Any]] +# ``(grid, launch_kwargs)`` or ``(grid, launch_kwargs, outputs)`` for +# self-allocating kernels. ``grid=None`` skips the launch (e.g. empty batch) +# but still returns the declared outputs. +LaunchSpec = ( + tuple[tuple[int, ...] | None, dict[str, Any]] + | tuple[tuple[int, ...] | None, dict[str, Any], Any] +) def triton_scalar_specialization_rep(value: int) -> int: @@ -91,6 +97,7 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]): kernel: ClassVar[Any] _warming = False + _warming_compile_key: CompileKeyT | None = None @abstractmethod def warmup_inputs(self, compile_key: CompileKeyT) -> dict[str, Any]: @@ -100,10 +107,12 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]): def compile(self, compile_key: CompileKeyT) -> None: inputs = self.warmup_inputs(compile_key) self._warming = True + self._warming_compile_key = compile_key try: cast(Callable[..., None], self)(**inputs) finally: self._warming = False + self._warming_compile_key = None @cached_property def _kernel_arg_names(self) -> tuple[str, ...]: @@ -119,11 +128,14 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]): def launch( self, - grid: tuple[int, ...], + grid: tuple[int, ...] | None, inputs: Mapping[str, Any], /, + *, + kernel: Any = None, **kwargs: Any, ) -> Any: + kernel = kernel if kernel is not None else self.kernel runtime_launcher = kwargs.pop("_runtime_launcher", None) runtime_launcher_arg_count = kwargs.pop("_runtime_launcher_arg_count", 0) for name, value in inputs.items(): @@ -131,22 +143,36 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]): if target in self._kernel_arg_names and target not in kwargs: kwargs[target] = value if self._warming: - warmup = getattr(self.kernel, "warmup", None) + if ( + self._warming_compile_key is not None + and "launch_pdl" in kwargs + and hasattr(self._warming_compile_key, "launch_pdl") + ): + kwargs["launch_pdl"] = self._warming_compile_key.launch_pdl + warmup = getattr(kernel, "warmup", None) assert warmup is not None return warmup(grid=(1,), **kwargs) + if grid is None: + return None if runtime_launcher is not None: regular_args = [ kwargs.pop(name) for name in self._kernel_arg_names[:runtime_launcher_arg_count] ] - return runtime_launcher(self.kernel, grid, *regular_args, **kwargs) - return self.kernel[grid](**kwargs) + return runtime_launcher(kernel, grid, *regular_args, **kwargs) + return kernel[grid](**kwargs) def kernel_launcher( call_fn: Callable[..., LaunchSpec], -) -> Callable[..., None]: - """Launch a Triton kernel from a declarative ``__call__`` specification.""" +) -> Callable[..., Any]: + """Launch a Triton kernel from a declarative ``__call__`` specification. + + ``call_fn`` returns either ``(grid, launch_kwargs)`` or, when it allocates + its own outputs, ``(grid, launch_kwargs, outputs)``. The declared outputs are + returned to the caller. A ``grid`` of ``None`` skips the launch (e.g. an + empty-token batch) while still returning the outputs. + """ signature = inspect.signature(call_fn) @wraps(call_fn) @@ -154,14 +180,20 @@ def kernel_launcher( self: VllmTritonJitKernel[Any], *args: Any, **kwargs: Any, - ) -> None: - grid, launch_kwargs = call_fn(self, *args, **kwargs) + ) -> Any: + spec = call_fn(self, *args, **kwargs) + if len(spec) == 3: + grid, launch_kwargs, outputs = spec + else: + grid, launch_kwargs = spec + outputs = None bound = signature.bind(self, *args, **kwargs) bound.apply_defaults() inputs = { name: value for name, value in bound.arguments.items() if name != "self" } self.launch(grid, inputs, **launch_kwargs) + return outputs return wrapper diff --git a/vllm/models/common/ops/__init__.py b/vllm/models/common/ops/__init__.py index 0d0fe8de26..0260b1fa1b 100644 --- a/vllm/models/common/ops/__init__.py +++ b/vllm/models/common/ops/__init__.py @@ -2,8 +2,9 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Ops shared across model implementations.""" -from .fused_qk_rmsnorm import fused_q_kv_rmsnorm +from .fused_qk_rmsnorm import _FUSED_Q_KV_RMSNORM_KERNEL, fused_q_kv_rmsnorm __all__ = [ + "_FUSED_Q_KV_RMSNORM_KERNEL", "fused_q_kv_rmsnorm", ] diff --git a/vllm/models/common/ops/fused_qk_rmsnorm.py b/vllm/models/common/ops/fused_qk_rmsnorm.py index 455b38d335..c9e3f04c92 100644 --- a/vllm/models/common/ops/fused_qk_rmsnorm.py +++ b/vllm/models/common/ops/fused_qk_rmsnorm.py @@ -1,67 +1,194 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from dataclasses import dataclass +from typing import Any + import torch +from vllm.model_executor.warmup.jit_warmup_triton_helper import ( + LaunchSpec, + TritonWarmupTensor, + VllmTritonJitKernel, + kernel_launcher, +) from vllm.platforms import current_platform from vllm.triton_utils import tl, triton +from vllm.utils.math_utils import next_power_of_2 -@triton.jit -def _fused_q_kv_rmsnorm_kernel( - q_ptr, - q_out_ptr, - q_weight_ptr, - q_in_stride, - q_out_stride, - kv_ptr, - kv_out_ptr, - kv_weight_ptr, - kv_in_stride, - kv_out_stride, - eps, - Q_SIZE: tl.constexpr, - KV_SIZE: tl.constexpr, - BLOCK_SIZE: tl.constexpr, - launch_pdl: tl.constexpr, -): - # num_tokens goes on grid-x (max 2**31 - 1); task goes on grid-y. - # CUDA's grid-y/z are capped at 65535, so putting num_tokens there crashes - # the launch at max-num-batched-tokens >= 65536 with "invalid argument". - # int64: q_in_stride can be ~24K (128 heads × 192) and overflows int32 - # past num_tokens ~87K under large chunked prefill. - token_idx = tl.program_id(0).to(tl.int64) - pid_task = tl.program_id(1) +class FusedQKVRMSNormKernel(VllmTritonJitKernel["FusedQKVRMSNormKernel.CompileKey"]): + @dataclass(frozen=True) + class CompileKey: + dtype: torch.dtype + q_size: int + kv_size: int + block_size: int + q_in_stride: int + q_out_stride: int + kv_in_stride: int + kv_out_stride: int + eps: float + launch_pdl: bool - if pid_task == 0: - SIZE = Q_SIZE - row_in = q_ptr + token_idx * q_in_stride - weight_ptr = q_weight_ptr - row_out = q_out_ptr + token_idx * q_out_stride - else: - SIZE = KV_SIZE - row_in = kv_ptr + token_idx * kv_in_stride - weight_ptr = kv_weight_ptr - row_out = kv_out_ptr + token_idx * kv_out_stride + @staticmethod + @triton.jit + def kernel( + q_ptr, + q_out_ptr, + q_weight_ptr, + q_in_stride, + q_out_stride, + kv_ptr, + kv_out_ptr, + kv_weight_ptr, + kv_in_stride, + kv_out_stride, + eps, + Q_SIZE: tl.constexpr, + KV_SIZE: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + launch_pdl: tl.constexpr, + ): + # num_tokens goes on grid-x (max 2**31 - 1); task goes on grid-y. + # CUDA's grid-y/z are capped at 65535, so putting num_tokens there crashes + # the launch at max-num-batched-tokens >= 65536 with "invalid argument". + # int64: q_in_stride can be ~24K (128 heads × 192) and overflows int32 + # past num_tokens ~87K under large chunked prefill. + token_idx = tl.program_id(0).to(tl.int64) + pid_task = tl.program_id(1) - # RMSNorm in fp32 throughout — matches csrc/layernorm_kernels.cu's - # `(scalar_t)(x * s_variance * w)` and DeepseekV4's compressor kernel, which - # keep x, rrms, and w all in fp32 and perform a single cast at store. - block = tl.arange(0, BLOCK_SIZE) - mask = block < SIZE - # The weight load does not depend on the producer's output, so issue it - # before the PDL wait: gamma streams in while the producer finishes, - # leaving a single dependent global round trip (x) after the wait. - w = tl.load(weight_ptr + block, mask=mask, other=0.0).to(tl.float32) + if pid_task == 0: + SIZE = Q_SIZE + row_in = q_ptr + token_idx * q_in_stride + weight_ptr = q_weight_ptr + row_out = q_out_ptr + token_idx * q_out_stride + else: + SIZE = KV_SIZE + row_in = kv_ptr + token_idx * kv_in_stride + weight_ptr = kv_weight_ptr + row_out = kv_out_ptr + token_idx * kv_out_stride - if launch_pdl: - tl.extra.cuda.gdc_wait() - tl.extra.cuda.gdc_launch_dependents() + # RMSNorm in fp32 throughout — matches csrc/layernorm_kernels.cu's + # `(scalar_t)(x * s_variance * w)` and DeepseekV4's compressor kernel, which + # keep x, rrms, and w all in fp32 and perform a single cast at store. + block = tl.arange(0, BLOCK_SIZE) + mask = block < SIZE + # The weight load does not depend on the producer's output, so issue it + # before the PDL wait: gamma streams in while the producer finishes, + # leaving a single dependent global round trip (x) after the wait. + w = tl.load(weight_ptr + block, mask=mask, other=0.0).to(tl.float32) - x = tl.load(row_in + block, mask=mask, other=0.0).to(tl.float32) - variance = tl.sum(x * x, axis=0) / SIZE - rrms = tl.rsqrt(variance + eps) - y = x * rrms * w - tl.store(row_out + block, y.to(row_out.dtype.element_ty), mask=mask) + if launch_pdl: + tl.extra.cuda.gdc_wait() + tl.extra.cuda.gdc_launch_dependents() + + x = tl.load(row_in + block, mask=mask, other=0.0).to(tl.float32) + variance = tl.sum(x * x, axis=0) / SIZE + rrms = tl.rsqrt(variance + eps) + y = x * rrms * w + tl.store(row_out + block, y.to(row_out.dtype.element_ty), mask=mask) + + def dispatch( # type: ignore[override] + self, + *, + dtype: torch.dtype, + q_size: int, + kv_size: int, + eps: float, + launch_pdl: bool, + **compile_key_fields: int, + ) -> CompileKey: + max_size = q_size if q_size >= kv_size else kv_size + return self.CompileKey( + **compile_key_fields, + dtype=dtype, + q_size=q_size, + kv_size=kv_size, + block_size=next_power_of_2(max_size), + eps=eps, + launch_pdl=launch_pdl, + ) + + def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]: + hf_config = vllm_config.model_config.hf_config + q_size = int(getattr(hf_config, "q_lora_rank", 0) or 0) + kv_size = int(getattr(hf_config, "head_dim", 0) or 0) + if q_size <= 0 or kv_size <= 0: + return [] + + input_stride = q_size + kv_size + return self._trace_dispatch(self.dispatch)( + dtype=vllm_config.model_config.dtype, + q_size=q_size, + kv_size=kv_size, + q_in_stride=input_stride, + q_out_stride=(input_stride, q_size), + kv_in_stride=input_stride, + kv_out_stride=(input_stride, kv_size), + eps=float(hf_config.rms_norm_eps), + launch_pdl=(False, True), + ) + + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + return dict( + q=TritonWarmupTensor( + compile_key.dtype, + shape=(1, compile_key.q_size), + strides=(compile_key.q_in_stride, 1), + ), + kv=TritonWarmupTensor( + compile_key.dtype, + shape=(1, compile_key.kv_size), + strides=(compile_key.kv_in_stride, 1), + ), + q_out=TritonWarmupTensor( + compile_key.dtype, + shape=(1, compile_key.q_size), + strides=(compile_key.q_out_stride, 1), + ), + kv_out=TritonWarmupTensor( + compile_key.dtype, + shape=(1, compile_key.kv_size), + strides=(compile_key.kv_out_stride, 1), + ), + q_weight=TritonWarmupTensor( + compile_key.dtype, + shape=(compile_key.q_size,), + ), + kv_weight=TritonWarmupTensor( + compile_key.dtype, + shape=(compile_key.kv_size,), + ), + eps=compile_key.eps, + ) + + @kernel_launcher + def __call__( + self, + q: torch.Tensor, + kv: torch.Tensor, + q_out: torch.Tensor, + kv_out: torch.Tensor, + q_weight: torch.Tensor, + kv_weight: torch.Tensor, + eps: float, + ) -> LaunchSpec: + q_size = q.shape[1] + kv_size = kv.shape[1] + block_size = next_power_of_2(max(q_size, kv_size)) + return (q.shape[0], 2), dict( + q_in_stride=q.stride(0), + q_out_stride=q_out.stride(0), + kv_in_stride=kv.stride(0), + kv_out_stride=kv_out.stride(0), + Q_SIZE=q_size, + KV_SIZE=kv_size, + BLOCK_SIZE=block_size, + launch_pdl=current_platform.is_arch_support_pdl(), + # One vectorized 128-bit access per thread for K3/DSv4-sized rows, + # instead of a serial per-thread chain at the default 4 warps. + num_warps=8 if block_size >= 2048 else 4, + ) def fused_q_kv_rmsnorm( @@ -78,40 +205,19 @@ def fused_q_kv_rmsnorm( assert qr.stride(-1) == 1 and kv.stride(-1) == 1 assert q_weight.is_contiguous() and kv_weight.is_contiguous() - q_size = qr.shape[1] - kv_size = kv.shape[1] - num_tokens = qr.shape[0] - # Allocate packed outputs rather than empty_like: qr/kv are typically - # column slices of a fused qkv_a projection, and for num_tokens == 1 - # empty_like preserves the parent's row stride (torch keeps strides of - # size-1 dims), e.g. (2112, 1) for a [1, 1536] q slice. Downstream - # dispatchers that require packed row-major inputs (e.g. the Kimi-K3 - # low-latency GEMM's _is_packed_row_major check) then reject the tensor - # and silently fall back to cuBLAS on every decode step. qr_out = torch.empty(qr.shape, dtype=qr.dtype, device=qr.device) kv_out = torch.empty(kv.shape, dtype=kv.dtype, device=kv.device) - if num_tokens == 0: - return qr_out, kv_out - - block_size = triton.next_power_of_2(max(q_size, kv_size)) - _fused_q_kv_rmsnorm_kernel[(num_tokens, 2)]( - qr, - qr_out, - q_weight, - qr.stride(0), - qr_out.stride(0), - kv, - kv_out, - kv_weight, - kv.stride(0), - kv_out.stride(0), - eps, - Q_SIZE=q_size, - KV_SIZE=kv_size, - BLOCK_SIZE=block_size, - launch_pdl=current_platform.is_arch_support_pdl(), - # One vectorized 128-bit access per thread for K3/DSv4-sized rows, - # instead of a serial per-thread chain at the default 4 warps. - num_warps=8 if block_size >= 2048 else 4, - ) + if qr.shape[0] > 0: + _FUSED_Q_KV_RMSNORM_KERNEL( + qr, + kv, + qr_out, + kv_out, + q_weight, + kv_weight, + eps, + ) return qr_out, kv_out + + +_FUSED_Q_KV_RMSNORM_KERNEL = FusedQKVRMSNormKernel() diff --git a/vllm/models/deepseek_v4/amd/mtp.py b/vllm/models/deepseek_v4/amd/mtp.py index 962865b4e1..cafb6bb0ed 100644 --- a/vllm/models/deepseek_v4/amd/mtp.py +++ b/vllm/models/deepseek_v4/amd/mtp.py @@ -39,9 +39,9 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.deepseek_mtp import SharedHead from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weight_name from vllm.model_executor.models.utils import maybe_prefix -from vllm.models.deepseek_v4.common.ops import ( - fused_mtp_input_rmsnorm, - mtp_shared_head_rmsnorm, +from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import ( + _FUSED_MTP_INPUT_RMSNORM_KERNEL, + _MTP_SHARED_HEAD_RMSNORM_KERNEL, ) from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors @@ -127,6 +127,10 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module): self.hc_head_op = HCHeadOp() + if vllm_config.kernel_config.enable_jit_warmup: + _FUSED_MTP_INPUT_RMSNORM_KERNEL.register_warmup() + _MTP_SHARED_HEAD_RMSNORM_KERNEL.register_warmup() + def forward( self, input_ids: torch.Tensor, @@ -143,7 +147,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module): -1, self.hc_mult, self.config.hidden_size ) # Fused: mask inputs at position 0 (not needed by MTP), enorm, hnorm. - inputs_embeds, previous_hidden_states = fused_mtp_input_rmsnorm( + inputs_embeds, previous_hidden_states = _FUSED_MTP_INPUT_RMSNORM_KERNEL( inputs_embeds, positions, previous_hidden_states, @@ -256,7 +260,7 @@ class DeepSeekV4MultiTokenPredictor(nn.Module): mtp_layer.rms_norm_eps, mtp_layer.hc_eps, ) - hidden_states = mtp_shared_head_rmsnorm( + hidden_states = _MTP_SHARED_HEAD_RMSNORM_KERNEL( hidden_states, mtp_layer.shared_head.norm.weight.data, mtp_layer.shared_head.norm.variance_epsilon, diff --git a/vllm/models/deepseek_v4/attention.py b/vllm/models/deepseek_v4/attention.py index d62e4d5bf5..12205bc043 100644 --- a/vllm/models/deepseek_v4/attention.py +++ b/vllm/models/deepseek_v4/attention.py @@ -22,7 +22,10 @@ from vllm.model_executor.layers.linear import ( RowParallelLinear, ) from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer -from vllm.models.common.ops import fused_q_kv_rmsnorm +from vllm.models.common.ops.fused_qk_rmsnorm import ( + _FUSED_Q_KV_RMSNORM_KERNEL, + fused_q_kv_rmsnorm, +) from vllm.models.deepseek_v4.common.ops import ( fused_indexer_q_rope_quant, ) @@ -417,6 +420,44 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): _COMBINE_TOPK_SWA_INDICES_KERNEL.register_warmup() + from vllm.platforms import current_platform + from vllm.utils.import_utils import has_cutedsl + + _FUSED_Q_KV_RMSNORM_KERNEL.register_warmup() + + backend_name = self.backend_cls.get_name() + if current_platform.is_cuda(): + from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import ( # noqa: E501 + _FUSED_INV_ROPE_FP8_QUANT_KERNEL, + ) + + _FUSED_INV_ROPE_FP8_QUANT_KERNEL.register_warmup() + + if self.compress_ratio == 128: + from vllm.models.deepseek_v4.sparse_mla import ( + _BUILD_C128A_TOPK_METADATA_KERNEL, + ) + + _BUILD_C128A_TOPK_METADATA_KERNEL.register_warmup() + + if backend_name == "FLASHMLA_SPARSE_DSV4": + from vllm.models.deepseek_v4.common.ops.cache_utils import ( + _COMPUTE_GLOBAL_TOPK_INDICES_AND_LENS_KERNEL, + _DEQUANTIZE_AND_GATHER_K_CACHE_KERNEL, + ) + + _COMPUTE_GLOBAL_TOPK_INDICES_AND_LENS_KERNEL.register_warmup() + if not has_cutedsl(): + _DEQUANTIZE_AND_GATHER_K_CACHE_KERNEL.register_warmup() + elif backend_name == "FLASHINFER_MLA_SPARSE_DSV4": + from vllm.models.deepseek_v4.common.ops.cache_utils import ( + _BUILD_FLASHINFER_MIXED_SPARSE_INDICES_KERNEL, + _COMPUTE_GLOBAL_TOPK_INDICES_AND_LENS_KERNEL, + ) + + _COMPUTE_GLOBAL_TOPK_INDICES_AND_LENS_KERNEL.register_warmup() + _BUILD_FLASHINFER_MIXED_SPARSE_INDICES_KERNEL.register_warmup() + def forward( self, positions: torch.Tensor, @@ -974,6 +1015,22 @@ class DeepseekV4Indexer(nn.Module): torch.cuda.Event(), ] + if vllm_config.kernel_config.enable_jit_warmup: + from vllm.platforms import current_platform + from vllm.utils.import_utils import has_cutedsl + + if not has_cutedsl() and not current_platform.is_xpu(): + from vllm.models.deepseek_v4.common.ops.fused_indexer_q import ( + _FUSED_INDEXER_Q_ROPE_MXFP4_TRITON_KERNEL, + _FUSED_INDEXER_Q_ROPE_QUANT_TRITON_KERNEL, + ) + + ( + _FUSED_INDEXER_Q_ROPE_MXFP4_TRITON_KERNEL + if self.use_fp4_kv + else _FUSED_INDEXER_Q_ROPE_QUANT_TRITON_KERNEL + ).register_warmup() + def forward( self, hidden_states: torch.Tensor, diff --git a/vllm/models/deepseek_v4/common/ops/__init__.py b/vllm/models/deepseek_v4/common/ops/__init__.py index b7899f01de..89209e7e41 100644 --- a/vllm/models/deepseek_v4/common/ops/__init__.py +++ b/vllm/models/deepseek_v4/common/ops/__init__.py @@ -10,8 +10,6 @@ from .cache_utils import ( ) from .fused_indexer_q import MXFP4_BLOCK_SIZE, fused_indexer_q_rope_quant from .fused_inv_rope_fp8_quant import fused_inv_rope_fp8_quant -from .fused_mtp_input_rmsnorm import fused_mtp_input_rmsnorm, mtp_shared_head_rmsnorm -from .save_partial_states import save_partial_states __all__ = [ "MXFP4_BLOCK_SIZE", @@ -21,8 +19,5 @@ __all__ = [ "dequantize_and_gather_k_cache", "fused_indexer_q_rope_quant", "fused_inv_rope_fp8_quant", - "fused_mtp_input_rmsnorm", - "mtp_shared_head_rmsnorm", "quantize_and_insert_k_cache", - "save_partial_states", ] diff --git a/vllm/models/deepseek_v4/common/ops/cache_utils.py b/vllm/models/deepseek_v4/common/ops/cache_utils.py index fefc10ec35..1bd0706f0f 100644 --- a/vllm/models/deepseek_v4/common/ops/cache_utils.py +++ b/vllm/models/deepseek_v4/common/ops/cache_utils.py @@ -22,10 +22,17 @@ import torch from vllm.model_executor.layers.quantization.utils.quant_utils import ( get_fp8_min_max, ) -from vllm.model_executor.warmup.jit_warmup import VllmJitKernel, zip_inputs +from vllm.model_executor.warmup.jit_warmup import ( + WarmupIntRange, + zip_inputs, +) from vllm.model_executor.warmup.jit_warmup_triton_helper import ( + LaunchSpec, TritonPointerInputVariant, TritonWarmupTensor, + VllmTritonJitKernel, + kernel_launcher, + triton_scalar_specialization_rep, ) from vllm.platforms import current_platform from vllm.triton_utils import tl, triton @@ -225,166 +232,268 @@ def quantize_and_insert_k_cache( ) -@triton.jit -def _dequantize_and_gather_k_kernel( - out_ptr, - out_stride0, - out_stride1, - k_cache_ptr, - seq_lens_ptr, - block_table_ptr, - offset, - gather_lens_ptr, - # Constants - max_blocks_per_seq: tl.constexpr, - fp8_dim: tl.constexpr, # 448 - bf16_dim: tl.constexpr, # 64 - scale_dim: tl.constexpr, # 8 - quant_block: tl.constexpr, # 64 (quantization block size) - cache_block_size: tl.constexpr, # 64 or 128 (paged cache block size) - token_data_size: tl.constexpr, # 576 bytes per token data - block_stride: tl.constexpr, # total bytes per block (padded) int32 - output_dim: tl.constexpr, # 512 - fp8_max: tl.constexpr, - n_quant_blocks: tl.constexpr, # 7 real blocks - use_fnuz: tl.constexpr = False, +class DequantizeAndGatherKCacheKernel( + VllmTritonJitKernel["DequantizeAndGatherKCacheKernel.CompileKey"] ): - batch_idx = tl.program_id(0) - worker_id = tl.program_id(1) - num_workers = tl.num_programs(1) + NUM_WORKERS = 128 - seq_len = tl.load(seq_lens_ptr + batch_idx) - if gather_lens_ptr is not None: # noqa: SIM108 - gather_len = tl.load(gather_lens_ptr + batch_idx) - else: - # Gather all tokens - gather_len = seq_len - start_pos = seq_len - gather_len + @dataclass(frozen=True) + class CompileKey: + max_blocks_per_seq: int + cache_block_size: int + block_stride: int + use_fnuz: bool + has_gather_lens: bool + offset: int - for i in range(worker_id, gather_len, num_workers): - # Calculate the actual token index in the sequence - pos = start_pos + i + @staticmethod + @triton.jit + def kernel( + out_ptr, + out_stride0, + out_stride1, + k_cache_ptr, + seq_lens_ptr, + block_table_ptr, + offset, + gather_lens_ptr, + # Constants + max_blocks_per_seq: tl.constexpr, + fp8_dim: tl.constexpr, # 448 + bf16_dim: tl.constexpr, # 64 + scale_dim: tl.constexpr, # 8 + quant_block: tl.constexpr, # 64 (quantization block size) + cache_block_size: tl.constexpr, # 64 or 128 (paged cache block size) + token_data_size: tl.constexpr, # 576 bytes per token data + block_stride: tl.constexpr, # total bytes per block (padded) int32 + output_dim: tl.constexpr, # 512 + fp8_max: tl.constexpr, + n_quant_blocks: tl.constexpr, # 7 real blocks + use_fnuz: tl.constexpr = False, + ): + batch_idx = tl.program_id(0) + worker_id = tl.program_id(1) + num_workers = tl.num_programs(1) - # Calculate which block and position within block - block_in_seq = pos // cache_block_size - pos_in_block = pos % cache_block_size + seq_len = tl.load(seq_lens_ptr + batch_idx) + if gather_lens_ptr is not None: # noqa: SIM108 + gather_len = tl.load(gather_lens_ptr + batch_idx) + else: + # Gather all tokens + gather_len = seq_len + start_pos = seq_len - gather_len - # Get physical block index from block table - block_table_row_ptr = block_table_ptr + batch_idx * max_blocks_per_seq - physical_block_idx = tl.load(block_table_row_ptr + block_in_seq) # int32 + for i in range(worker_id, gather_len, num_workers): + # Calculate the actual token index in the sequence + pos = start_pos + i - # int64: physical_block_idx * block_stride can exceed 2^31 with many - # KV-cache blocks (e.g. >= 57K at block_stride ~37K). - cache_block_ptr = k_cache_ptr + physical_block_idx.to(tl.int64) * block_stride + # Calculate which block and position within block + block_in_seq = pos // cache_block_size + pos_in_block = pos % cache_block_size - # Token data pointer - token_data_ptr = cache_block_ptr + pos_in_block * token_data_size + # Get physical block index from block table + block_table_row_ptr = block_table_ptr + batch_idx * max_blocks_per_seq + physical_block_idx = tl.load(block_table_row_ptr + block_in_seq) # int32 - # Scale pointer: after all token data - token_scale_ptr = ( - cache_block_ptr - + cache_block_size * token_data_size - + pos_in_block * scale_dim + # int64: physical_block_idx * block_stride can exceed 2^31 with many + # KV-cache blocks (e.g. >= 57K at block_stride ~37K). + cache_block_ptr = ( + k_cache_ptr + physical_block_idx.to(tl.int64) * block_stride + ) + + # Token data pointer + token_data_ptr = cache_block_ptr + pos_in_block * token_data_size + + # Scale pointer: after all token data + token_scale_ptr = ( + cache_block_ptr + + cache_block_size * token_data_size + + pos_in_block * scale_dim + ) + + # Token data layout: [0:448] fp8, [448:576] bf16 + token_fp8_ptr = token_data_ptr + token_bf16_ptr = token_data_ptr + fp8_dim + + # Output pointer for this token (flattened) + output_row_ptr = ( + out_ptr + batch_idx * out_stride0 + (offset + i) * out_stride1 + ) + + # ========== Dequantize FP8 portion using UE8M0 ========== + for qblock_idx in tl.static_range(n_quant_blocks): + qblock_start = qblock_idx * quant_block + + if qblock_start < fp8_dim: + offsets = qblock_start + tl.arange(0, quant_block) + mask = offsets < fp8_dim + + # Load quantized fp8 values (stored as uint8) + x_uint8 = tl.load(token_fp8_ptr + offsets, mask=mask, other=0) + + # Bitcast uint8 back to fp8 (FNUZ on gfx942, OCP elsewhere). + if use_fnuz: + x_fp8 = x_uint8.to(tl.float8e4b8, bitcast=True) + else: + x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True) + + # Convert fp8 to float32 for computation + x_float = x_fp8.to(tl.float32) + + # Load and decode UE8M0 scale + # UE8M0: scale = 2^(stored_value - 127) + encoded_scale = tl.load(token_scale_ptr + qblock_idx) + exponent = encoded_scale.to(tl.float32) - 127.0 + scale = tl.exp2(exponent) + + # Dequantize: bf16_value = fp8_value * scale + x_dequant = x_float * scale + + # Store as bf16 + tl.store( + output_row_ptr + offsets, x_dequant.to(tl.bfloat16), mask=mask + ) + + # ========== Copy BF16 portion directly ========== + bf16_output_offset = fp8_dim # After 448 elements in output + + # Read bf16 from cache + bf16_cache_ptr = token_bf16_ptr.to(tl.pointer_type(tl.bfloat16)) + + # Process in chunks of 16 + for j in tl.static_range(bf16_dim // 16): + chunk_offsets = j * 16 + tl.arange(0, 16) + bf16_vals = tl.load(bf16_cache_ptr + chunk_offsets) + tl.store(output_row_ptr + bf16_output_offset + chunk_offsets, bf16_vals) + + def dispatch( # type: ignore[override] + self, + *, + max_model_len: int, + block_table_block_size: int, + cache_block_size: int, + offset: int, + **compile_key_fields: bool, + ) -> CompileKey: + token_stride = 576 + scale_stride = 8 + unpadded = cache_block_size * (token_stride + scale_stride) + block_stride = ((unpadded + token_stride - 1) // token_stride) * token_stride + return self.CompileKey( + **compile_key_fields, + max_blocks_per_seq=(max_model_len + block_table_block_size - 1) + // block_table_block_size, + cache_block_size=cache_block_size, + block_stride=block_stride, + offset=triton_scalar_specialization_rep(offset), ) - # Token data layout: [0:448] fp8, [448:576] bf16 - token_fp8_ptr = token_data_ptr - token_bf16_ptr = token_data_ptr + fp8_dim + def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]: + max_model_len = vllm_config.model_config.max_model_len + block_size = vllm_config.cache_config.block_size + if max_model_len <= 0 or block_size <= 0: + return [] - # Output pointer for this token (flattened) - output_row_ptr = out_ptr + batch_idx * out_stride0 + (offset + i) * out_stride1 + compress_ratios = frozenset( + max(1, int(compress_ratio)) + for compress_ratio in vllm_config.model_config.hf_config.compress_ratios + ) + return self._trace_dispatch(self.dispatch)( + zip_inputs( + dict( + cache_block_size=block_size, + has_gather_lens=True, + use_fnuz=current_platform.is_fp8_fnuz(), + offset=1, + enabled=True, + ), + dict( + cache_block_size=block_size, + has_gather_lens=True, + use_fnuz=current_platform.is_fp8_fnuz(), + offset=2, + enabled=True, + ), + dict( + cache_block_size=block_size, + has_gather_lens=True, + use_fnuz=current_platform.is_fp8_fnuz(), + offset=16, + enabled=True, + ), + dict( + cache_block_size=max(block_size // 4, 1), + has_gather_lens=False, + use_fnuz=False, + offset=16, + enabled=4 in compress_ratios, + ), + dict( + cache_block_size=max(block_size // 128, 1), + has_gather_lens=False, + use_fnuz=False, + offset=16, + enabled=128 in compress_ratios, + ), + ), + max_model_len=max_model_len, + block_table_block_size=block_size, + _when=lambda *, enabled: enabled, + ) - # ========== Dequantize FP8 portion using UE8M0 ========== - for qblock_idx in tl.static_range(n_quant_blocks): - qblock_start = qblock_idx * quant_block + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + int32_ptr = TritonWarmupTensor(torch.int32) + return dict( + out=TritonWarmupTensor( + torch.bfloat16, + shape=(1, 1, 512), + strides=(512, 512, 1), + ), + k_cache=TritonWarmupTensor( + torch.uint8, + shape=(1, 1), + strides=(compile_key.block_stride, 1), + ), + seq_lens=int32_ptr, + gather_lens=(int32_ptr if compile_key.has_gather_lens else None), + block_table=TritonWarmupTensor( + torch.int32, + shape=(1, compile_key.max_blocks_per_seq), + ), + block_size=compile_key.cache_block_size, + offset=compile_key.offset, + use_fnuz=compile_key.use_fnuz, + ) - if qblock_start < fp8_dim: - offsets = qblock_start + tl.arange(0, quant_block) - mask = offsets < fp8_dim - - # Load quantized fp8 values (stored as uint8) - x_uint8 = tl.load(token_fp8_ptr + offsets, mask=mask, other=0) - - # Bitcast uint8 back to fp8 (FNUZ on gfx942, OCP elsewhere). - if use_fnuz: - x_fp8 = x_uint8.to(tl.float8e4b8, bitcast=True) - else: - x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True) - - # Convert fp8 to float32 for computation - x_float = x_fp8.to(tl.float32) - - # Load and decode UE8M0 scale - # UE8M0: scale = 2^(stored_value - 127) - encoded_scale = tl.load(token_scale_ptr + qblock_idx) - exponent = encoded_scale.to(tl.float32) - 127.0 - scale = tl.exp2(exponent) - - # Dequantize: bf16_value = fp8_value * scale - x_dequant = x_float * scale - - # Store as bf16 - tl.store(output_row_ptr + offsets, x_dequant.to(tl.bfloat16), mask=mask) - - # ========== Copy BF16 portion directly ========== - bf16_output_offset = fp8_dim # After 448 elements in output - - # Read bf16 from cache - bf16_cache_ptr = token_bf16_ptr.to(tl.pointer_type(tl.bfloat16)) - - # Process in chunks of 16 - for j in tl.static_range(bf16_dim // 16): - chunk_offsets = j * 16 + tl.arange(0, 16) - bf16_vals = tl.load(bf16_cache_ptr + chunk_offsets) - tl.store(output_row_ptr + bf16_output_offset + chunk_offsets, bf16_vals) - - -def dequantize_and_gather_k_cache_triton( - # [num_reqs, max_num_tokens, head_size] - out: torch.Tensor, - # [num_blocks, block_size, head_bytes] - k_cache: torch.Tensor, - # [num_reqs] - seq_lens: torch.Tensor, - # [num_reqs] - gather_lens: torch.Tensor | None, - # [num_reqs, max_blocks_per_seq] - block_table: torch.Tensor, - block_size: int, - offset: int, - use_fnuz: bool = False, -) -> None: - TOKEN_FP8_DIM = 448 - TOKEN_BF16_DIM = 64 - TOKEN_SCALE_DIM = 8 - QUANT_BLOCK_SIZE = 64 - FP8_MAX = 448.0 - TOKEN_DATA_SIZE = TOKEN_FP8_DIM + TOKEN_BF16_DIM * 2 - - num_reqs = seq_lens.shape[0] - NUM_WORKERS = 128 - _dequantize_and_gather_k_kernel[(num_reqs, NUM_WORKERS)]( - out, - out.stride(0), - out.stride(1), - k_cache, - seq_lens, - block_table, - offset, - gather_lens, - max_blocks_per_seq=block_table.shape[-1], - fp8_dim=TOKEN_FP8_DIM, - bf16_dim=TOKEN_BF16_DIM, - scale_dim=TOKEN_SCALE_DIM, - quant_block=QUANT_BLOCK_SIZE, - cache_block_size=block_size, - token_data_size=TOKEN_DATA_SIZE, - block_stride=k_cache.stride(0), - output_dim=512, - fp8_max=FP8_MAX, - n_quant_blocks=7, - use_fnuz=use_fnuz, - ) + @kernel_launcher + def __call__( + self, + out: torch.Tensor, + k_cache: torch.Tensor, + seq_lens: torch.Tensor, + gather_lens: torch.Tensor | None, + block_table: torch.Tensor, + block_size: int, + offset: int, + *, + use_fnuz: bool = False, + ) -> LaunchSpec: + num_reqs = seq_lens.shape[0] + return (num_reqs, self.NUM_WORKERS), dict( + out_stride0=out.stride(0), + out_stride1=out.stride(1), + max_blocks_per_seq=block_table.shape[-1], + fp8_dim=448, + bf16_dim=64, + scale_dim=8, + quant_block=64, + cache_block_size=block_size, + token_data_size=576, + block_stride=k_cache.stride(0), + output_dim=512, + fp8_max=448.0, + n_quant_blocks=7, + ) def dequantize_and_gather_k_cache( @@ -420,7 +529,7 @@ def dequantize_and_gather_k_cache( ) return - dequantize_and_gather_k_cache_triton( + _DEQUANTIZE_AND_GATHER_K_CACHE_KERNEL( out, k_cache, seq_lens, @@ -449,72 +558,181 @@ def compute_global_topk_indices_and_lens( num_tokens = topk_indices.shape[0] global_topk_indices = torch.empty_like(topk_indices) topk_lens = torch.empty(num_tokens, dtype=torch.int32, device=topk_indices.device) - _compute_global_topk_indices_and_lens_kernel[(num_tokens,)]( + _COMPUTE_GLOBAL_TOPK_INDICES_AND_LENS_KERNEL( global_topk_indices, - global_topk_indices.stride(0), topk_lens, topk_indices, - topk_indices.stride(0), - topk_indices.shape[-1], token_to_req_indices, block_table, - block_table.stride(0), block_size, is_valid_token, - TRITON_BLOCK_SIZE=1024, ) return global_topk_indices, topk_lens -@triton.jit -def _compute_global_topk_indices_and_lens_kernel( - global_topk_indices_ptr, - global_topk_indices_stride: tl.constexpr, - topk_lens_ptr, - topk_indices_ptr, - topk_indices_stride: tl.constexpr, - topk: tl.constexpr, - token_to_req_indices_ptr, - block_table_ptr, - block_table_stride: tl.constexpr, - block_size: tl.constexpr, - is_valid_token_ptr, - TRITON_BLOCK_SIZE: tl.constexpr, +class ComputeGlobalTopkIndicesAndLensKernel( + VllmTritonJitKernel["ComputeGlobalTopkIndicesAndLensKernel.CompileKey"] ): - token_idx = tl.program_id(0) - is_valid_token = tl.load(is_valid_token_ptr + token_idx) - req_idx = tl.load(token_to_req_indices_ptr + token_idx) + TRITON_BLOCK_SIZE = 1024 - count = tl.zeros((), dtype=tl.int32) - for i in range(0, topk, TRITON_BLOCK_SIZE): - offset = i + tl.arange(0, TRITON_BLOCK_SIZE) - mask = offset < topk + @dataclass(frozen=True) + class CompileKey: + global_topk_indices_stride: int + topk_indices_stride: int + topk: int + block_table_stride: int + block_size: int - local_idx = tl.load( - topk_indices_ptr + token_idx * topk_indices_stride + offset, - mask=mask, - other=-1, + @staticmethod + @triton.jit + def kernel( + global_topk_indices_ptr, + global_topk_indices_stride: tl.constexpr, + topk_lens_ptr, + topk_indices_ptr, + topk_indices_stride: tl.constexpr, + topk: tl.constexpr, + token_to_req_indices_ptr, + block_table_ptr, + block_table_stride: tl.constexpr, + block_size: tl.constexpr, + is_valid_token_ptr, + TRITON_BLOCK_SIZE: tl.constexpr, + ): + token_idx = tl.program_id(0) + is_valid_token = tl.load(is_valid_token_ptr + token_idx) + req_idx = tl.load(token_to_req_indices_ptr + token_idx) + + count = tl.zeros((), dtype=tl.int32) + for i in range(0, topk, TRITON_BLOCK_SIZE): + offset = i + tl.arange(0, TRITON_BLOCK_SIZE) + mask = offset < topk + + local_idx = tl.load( + topk_indices_ptr + token_idx * topk_indices_stride + offset, + mask=mask, + other=-1, + ) + is_valid = local_idx >= 0 + + block_indices = local_idx // block_size + block_numbers = tl.load( + block_table_ptr + req_idx * block_table_stride + block_indices, + mask=mask & is_valid, + ) + block_offsets = local_idx % block_size + + slot_ids = block_numbers * block_size + block_offsets + slot_ids = tl.where(is_valid, slot_ids, -1) + tl.store( + global_topk_indices_ptr + + token_idx * global_topk_indices_stride + + offset, + slot_ids, + mask=mask, + ) + count += tl.sum(is_valid.to(tl.int32), axis=0) + + # Zero out length for padding tokens. + tl.store(topk_lens_ptr + token_idx, tl.where(is_valid_token, count, 0)) + + def dispatch( # type: ignore[override] + self, + *, + topk_width: int, + block_size: int, + block_table_block_size: int, + max_model_len: int, + ) -> CompileKey: + return self.CompileKey( + global_topk_indices_stride=topk_width, + topk_indices_stride=topk_width, + topk=topk_width, + block_table_stride=(max_model_len + block_table_block_size - 1) + // block_table_block_size, + block_size=block_size, ) - is_valid = local_idx >= 0 - block_indices = local_idx // block_size - block_numbers = tl.load( - block_table_ptr + req_idx * block_table_stride + block_indices, - mask=mask & is_valid, + def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]: + index_topk = vllm_config.model_config.hf_config.index_topk + compress_ratios = vllm_config.model_config.hf_config.compress_ratios + cache_block_size = vllm_config.cache_config.block_size + if index_topk <= 0 or cache_block_size <= 0: + return [] + + max_c128a_topk = ( + ((vllm_config.model_config.max_model_len + 127) // 128 + 127) // 128 * 128 ) - block_offsets = local_idx % block_size - - slot_ids = block_numbers * block_size + block_offsets - slot_ids = tl.where(is_valid, slot_ids, -1) - tl.store( - global_topk_indices_ptr + token_idx * global_topk_indices_stride + offset, - slot_ids, - mask=mask, + active_c128a_topk_widths = WarmupIntRange( + 128, + max_c128a_topk + 1, + advance=lambda width: ( + min(width * 2, max_c128a_topk) + if width < max_c128a_topk + else max_c128a_topk + 1 + ), ) - count += tl.sum(is_valid.to(tl.int32), axis=0) + trace_dispatch = self._trace_dispatch(self.dispatch) + c4_keys = trace_dispatch( + topk_width=index_topk if 4 in compress_ratios else (), + block_size=max(1, cache_block_size // 4), + block_table_block_size=cache_block_size, + max_model_len=vllm_config.model_config.max_model_len, + ) + c128a_keys = trace_dispatch( + topk_width=active_c128a_topk_widths if 128 in compress_ratios else (), + block_size=max(1, cache_block_size // 128), + block_table_block_size=cache_block_size, + max_model_len=vllm_config.model_config.max_model_len, + ) + return list(dict.fromkeys((*c4_keys, *c128a_keys))) - # Zero out length for padding tokens. - tl.store(topk_lens_ptr + token_idx, tl.where(is_valid_token, count, 0)) + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + int32_ptr = TritonWarmupTensor(torch.int32) + return dict( + global_topk_indices=TritonWarmupTensor( + torch.int32, + shape=(1, compile_key.topk), + strides=(compile_key.global_topk_indices_stride, 1), + ), + topk_lens=int32_ptr, + topk_indices=TritonWarmupTensor( + torch.int32, + shape=(1, compile_key.topk), + strides=(compile_key.topk_indices_stride, 1), + ), + token_to_req_indices=int32_ptr, + block_table=TritonWarmupTensor( + torch.int32, + shape=(1, compile_key.block_table_stride), + ), + block_size=compile_key.block_size, + is_valid_token=TritonWarmupTensor(torch.bool), + ) + + @kernel_launcher + def __call__( + self, + global_topk_indices: torch.Tensor, + topk_lens: torch.Tensor, + topk_indices: torch.Tensor, + token_to_req_indices: torch.Tensor, + block_table: torch.Tensor, + block_size: int, + is_valid_token: torch.Tensor, + ) -> LaunchSpec: + num_tokens = topk_indices.shape[0] + return (num_tokens,), dict( + global_topk_indices_stride=global_topk_indices.stride(0), + topk_indices_stride=topk_indices.stride(0), + topk=topk_indices.shape[-1], + block_table_stride=block_table.stride(0), + TRITON_BLOCK_SIZE=self.TRITON_BLOCK_SIZE, + ) + + +_DEQUANTIZE_AND_GATHER_K_CACHE_KERNEL = DequantizeAndGatherKCacheKernel() +_COMPUTE_GLOBAL_TOPK_INDICES_AND_LENS_KERNEL = ComputeGlobalTopkIndicesAndLensKernel() # FlashMLA sparse prefill asserts `params.topk % B_TOPK == 0` (see @@ -582,43 +800,6 @@ def combine_topk_swa_indices( return combined_indices, combined_lens -_COMBINE_TOPK_SWA_NUM_WORKERS = 256 - - -# Representative pointer alignment variants for Triton pointer specialization. -_COMBINE_TOPK_SWA_POINTER_INPUTS = zip_inputs( - dict( - topk_indices=True, - query_start_loc=True, - seq_lens=True, - gather_lens=True, - ), - dict( - topk_indices=True, - query_start_loc=False, - seq_lens=False, - gather_lens=True, - ), - dict( - topk_indices=False, - query_start_loc=False, - seq_lens=False, - gather_lens=False, - ), -) - - -_DSV4_COMBINE_TOPK_SWA_WARMUP_INPUTS = zip_inputs( - # DSv4-Flash / SWA-only and C4A. - dict(compress_ratio=1, topk=0, topk_width=512), - dict(compress_ratio=4, topk=512, topk_width=512), - # DSv4-Pro C4A. - dict(compress_ratio=4, topk=1024, topk_width=1024), - # DSv4-Pro C128A. - dict(compress_ratio=128, topk=8192, topk_width=8192), -) - - def _hf_config_int(vllm_config: Any, name: str, default: int) -> int: model_config = getattr(vllm_config, "model_config", None) hf_config = getattr(model_config, "hf_config", None) @@ -631,8 +812,10 @@ def _scheduler_config_int(vllm_config: Any, name: str, default: int) -> int: class CombineTopkSwaIndicesKernel( - VllmJitKernel["CombineTopkSwaIndicesKernel.CompileKey"] + VllmTritonJitKernel["CombineTopkSwaIndicesKernel.CompileKey"] ): + NUM_WORKERS = 256 + @dataclass(frozen=True) class CompileKey: TOP_K: int @@ -746,11 +929,14 @@ class CombineTopkSwaIndicesKernel( query_start_loc: bool, seq_lens: bool, gather_lens: bool, - topk: int, compress_ratio: int, + index_topk: int, + active_topk_width: int, WINDOW_SIZE: int, image_width: int = 0, ) -> CompileKey: + topk_width = active_topk_width if compress_ratio == 128 else index_topk + topk = 0 if compress_ratio == 1 else topk_width padded_topk = next_power_of_2(topk_width) input_variant = TritonPointerInputVariant.from_alignment( topk_indices=topk_indices, @@ -772,6 +958,23 @@ class CombineTopkSwaIndicesKernel( return [] window_size = _hf_config_int(vllm_config, "sliding_window", 128) + hf_config = vllm_config.model_config.hf_config + index_topk = int(getattr(hf_config, "index_topk", 0) or 0) + compress_ratios = tuple( + sorted({max(1, int(ratio)) for ratio in hf_config.compress_ratios}) + ) + max_c128a_topk = ( + ((vllm_config.model_config.max_model_len + 127) // 128 + 127) // 128 * 128 + ) + active_topk_widths = WarmupIntRange( + 128, + max_c128a_topk + 1, + advance=lambda width: ( + min(width * 2, max_c128a_topk) + if width < max_c128a_topk + else max_c128a_topk + 1 + ), + ) image_width = ( _hf_config_int(vllm_config, "vision_max_n_token", 0) if _hf_config_int(vllm_config, "vision_n_layers", 0) > 0 @@ -781,38 +984,59 @@ class CombineTopkSwaIndicesKernel( # and the in-image bidirectional variant. image_widths = [0, image_width] if image_width > 0 else [0] return self._trace_dispatch(self.dispatch)( - _DSV4_COMBINE_TOPK_SWA_WARMUP_INPUTS, - _COMBINE_TOPK_SWA_POINTER_INPUTS, + zip_inputs( + dict( + topk_indices=True, + query_start_loc=True, + seq_lens=True, + gather_lens=True, + ), + dict( + topk_indices=True, + query_start_loc=False, + seq_lens=False, + gather_lens=True, + ), + dict( + topk_indices=False, + query_start_loc=False, + seq_lens=False, + gather_lens=False, + ), + ), + compress_ratio=compress_ratios, + index_topk=index_topk, + active_topk_width=active_topk_widths, WINDOW_SIZE=window_size, image_width=image_widths, ) - def compile(self, compile_key: CompileKey) -> None: - warmup = getattr(self.kernel, "warmup", None) - assert warmup is not None + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: int32_ptr = TritonWarmupTensor(torch.int32) - input_variant = compile_key.input_variant - warmup( - int32_ptr, - 1, # do not specialize combined_indices_stride - int32_ptr, - input_variant.pointer("topk_indices", torch.int32), - 1, # do not specialize topk_indices_stride - input_variant.pointer("query_start_loc", torch.int32), - input_variant.pointer("seq_lens", torch.int32), - input_variant.pointer("gather_lens", torch.int32), - int32_ptr, - int32_ptr, - 1, # do not specialize M - 1, # do not specialize N + variant = compile_key.input_variant + has_image = compile_key.IMAGE_WIDTH > 0 + return dict( + combined_indices=int32_ptr, + combined_lens=int32_ptr, + topk_indices=variant.pointer( + "topk_indices", torch.int32, shape=(1, compile_key.PADDED_TOP_K) + ), + query_start_loc=variant.pointer("query_start_loc", torch.int32), + seq_lens=variant.pointer("seq_lens", torch.int32), + gather_lens=variant.pointer("gather_lens", torch.int32), + M=1, + N=1, TOP_K=compile_key.TOP_K, COMPRESS_RATIO=compile_key.COMPRESS_RATIO, WINDOW_SIZE=compile_key.WINDOW_SIZE, - IMAGE_WIDTH=compile_key.IMAGE_WIDTH, - PADDED_TOP_K=compile_key.PADDED_TOP_K, - grid=(1, _COMBINE_TOPK_SWA_NUM_WORKERS), + # Reproduce the runtime substitution: no-image batches launch with + # ``topk_indices`` in the visibility pointer slots (see __call__). + left_visible=(int32_ptr if has_image else None), + right_visible=(int32_ptr if has_image else None), + max_image_tokens=compile_key.IMAGE_WIDTH, ) + @kernel_launcher def __call__( self, combined_indices: torch.Tensor, @@ -830,23 +1054,15 @@ class CombineTopkSwaIndicesKernel( left_visible: torch.Tensor | None = None, right_visible: torch.Tensor | None = None, max_image_tokens: int = 0, - ) -> None: + ) -> LaunchSpec: num_reqs = seq_lens.shape[0] has_image = left_visible is not None image_width = max_image_tokens if has_image else 0 - self.kernel[(num_reqs, _COMBINE_TOPK_SWA_NUM_WORKERS)]( - combined_indices, - combined_indices.stride(0), - combined_lens, - topk_indices, - topk_indices.stride(0), - query_start_loc, - seq_lens, - gather_lens, - left_visible if has_image else topk_indices, - right_visible if has_image else topk_indices, - M, - N, + return (num_reqs, self.NUM_WORKERS), dict( + combined_indices_stride=combined_indices.stride(0), + topk_indices_stride=topk_indices.stride(0), + left_visible_ptr=left_visible if has_image else topk_indices, + right_visible_ptr=right_visible if has_image else topk_indices, TOP_K=TOP_K, COMPRESS_RATIO=COMPRESS_RATIO, WINDOW_SIZE=WINDOW_SIZE, @@ -974,44 +1190,38 @@ def build_flashinfer_mixed_sparse_indices( if compressed_block_span is None else compressed_block_span ) - _build_flashinfer_mixed_sparse_indices_kernel[(num_tokens,)]( + _BUILD_FLASHINFER_MIXED_SPARSE_INDICES_KERNEL( sparse_indices, - sparse_indices.stride(0), sparse_topk_lens, decode_swa_indices, - decode_swa_indices.stride(0), decode_compressed_indices, - decode_compressed_indices.stride(0), decode_compressed_topk_lens, decode_is_valid_token, prefill_topk_indices, - prefill_topk_indices.stride(0), prefill_left_visible if has_image else token_to_req_indices, prefill_right_visible if has_image else token_to_req_indices, query_start_loc, seq_lens, token_to_req_indices, swa_block_table, - swa_block_table.stride(0), swa_block_size, swa_span, compressed_block_table, - compressed_block_table.stride(0), compressed_block_size, compressed_span, - NUM_DECODE_TOKENS=num_decode_tokens, - WINDOW_SIZE=window_size, - SWA_INDEX_WIDTH=swa_index_width, - IMAGE_WIDTH=image_width, - COMPRESS_RATIO=compress_ratio, - TOP_K=topk, - PADDED_TOP_K=padded_topk, - PREFILL_TOPK_STRIDE=prefill_topk_indices.shape[-1], - DECODE_COMPRESSED_TOPK=decode_compressed_topk, - DECODE_COMPRESSED_INDICES_ARE_LOCAL=decode_compressed_indices_are_local, - HAS_DECODE_COMPRESSED_LENS=has_decode_compressed_lens, - WINDOW_BLOCK_SIZE=window_block_size, - TOPK_BLOCK_SIZE=topk_block_size, + num_decode_tokens=num_decode_tokens, + window_size=window_size, + swa_index_width=swa_index_width, + image_width=image_width, + compress_ratio=compress_ratio, + topk=topk, + padded_topk=padded_topk, + prefill_topk_stride=prefill_topk_indices.shape[-1], + decode_compressed_topk=decode_compressed_topk, + decode_compressed_indices_are_local=decode_compressed_indices_are_local, + has_decode_compressed_lens=has_decode_compressed_lens, + window_block_size=window_block_size, + topk_block_size=topk_block_size, num_warps=num_warps, ) return sparse_indices, sparse_topk_lens @@ -1029,205 +1239,444 @@ def _remap_flashinfer_index(values, block_size, block_span): return tl.where(is_valid, values, -1) -@triton.jit( - do_not_specialize=[ - "sparse_indices_stride", - "decode_swa_stride", - "decode_compressed_stride", - "prefill_topk_stride", - "swa_block_table_stride", - "swa_block_size", - "swa_block_span", - "compressed_block_table_stride", - "compressed_block_size", - "compressed_block_span", - "NUM_DECODE_TOKENS", - "PREFILL_TOPK_STRIDE", - ] -) -def _build_flashinfer_mixed_sparse_indices_kernel( - sparse_indices_ptr, - sparse_indices_stride, - sparse_topk_lens_ptr, - decode_swa_indices_ptr, - decode_swa_stride, - decode_compressed_indices_ptr, - decode_compressed_stride, - decode_compressed_topk_lens_ptr, - decode_is_valid_token_ptr, - prefill_topk_indices_ptr, - prefill_topk_stride, - left_visible_ptr, - right_visible_ptr, - query_start_loc_ptr, - seq_lens_ptr, - token_to_req_indices_ptr, - swa_block_table_ptr, - swa_block_table_stride, - swa_block_size, - swa_block_span, - compressed_block_table_ptr, - compressed_block_table_stride, - compressed_block_size, - compressed_block_span, - NUM_DECODE_TOKENS, - WINDOW_SIZE: tl.constexpr, - SWA_INDEX_WIDTH: tl.constexpr, - IMAGE_WIDTH: tl.constexpr, - COMPRESS_RATIO: tl.constexpr, - TOP_K: tl.constexpr, - PADDED_TOP_K: tl.constexpr, - PREFILL_TOPK_STRIDE, - DECODE_COMPRESSED_TOPK: tl.constexpr, - DECODE_COMPRESSED_INDICES_ARE_LOCAL: tl.constexpr, - HAS_DECODE_COMPRESSED_LENS: tl.constexpr, - WINDOW_BLOCK_SIZE: tl.constexpr, - TOPK_BLOCK_SIZE: tl.constexpr, +class BuildFlashinferMixedSparseIndicesKernel( + VllmTritonJitKernel["BuildFlashinferMixedSparseIndicesKernel.CompileKey"] ): - token_idx = tl.program_id(0) - # Total SWA column count; > SWA_INDEX_WIDTH only for in-image - # bidirectional visibility (vision variant), where the extra columns are - # -1-padded for decode rows. - SWA_TOTAL_WIDTH: tl.constexpr = SWA_INDEX_WIDTH + IMAGE_WIDTH + @dataclass(frozen=True) + class CompileKey: + window_size: int + swa_index_width: int + image_width: int + compress_ratio: int + top_k: int + padded_top_k: int + decode_compressed_topk: int + decode_compressed_indices_are_local: bool + has_decode_compressed_lens: bool + window_block_size: int + topk_block_size: int + + @staticmethod + @triton.jit( + do_not_specialize=[ + "sparse_indices_stride", + "decode_swa_stride", + "decode_compressed_stride", + "prefill_topk_stride", + "swa_block_table_stride", + "swa_block_size", + "swa_block_span", + "compressed_block_table_stride", + "compressed_block_size", + "compressed_block_span", + "NUM_DECODE_TOKENS", + "PREFILL_TOPK_STRIDE", + ] + ) + def kernel( + sparse_indices_ptr, + sparse_indices_stride, + sparse_topk_lens_ptr, + decode_swa_indices_ptr, + decode_swa_stride, + decode_compressed_indices_ptr, + decode_compressed_stride, + decode_compressed_topk_lens_ptr, + decode_is_valid_token_ptr, + prefill_topk_indices_ptr, + prefill_topk_stride, + left_visible_ptr, + right_visible_ptr, + query_start_loc_ptr, + seq_lens_ptr, + token_to_req_indices_ptr, + swa_block_table_ptr, + swa_block_table_stride, + swa_block_size, + swa_block_span, + compressed_block_table_ptr, + compressed_block_table_stride, + compressed_block_size, + compressed_block_span, + NUM_DECODE_TOKENS, + WINDOW_SIZE: tl.constexpr, + SWA_INDEX_WIDTH: tl.constexpr, + IMAGE_WIDTH: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, + TOP_K: tl.constexpr, + PADDED_TOP_K: tl.constexpr, + PREFILL_TOPK_STRIDE, + DECODE_COMPRESSED_TOPK: tl.constexpr, + DECODE_COMPRESSED_INDICES_ARE_LOCAL: tl.constexpr, + HAS_DECODE_COMPRESSED_LENS: tl.constexpr, + WINDOW_BLOCK_SIZE: tl.constexpr, + TOPK_BLOCK_SIZE: tl.constexpr, + ): + token_idx = tl.program_id(0) + # Total SWA column count; > SWA_INDEX_WIDTH only for in-image + # bidirectional visibility (vision variant), where the extra columns are + # -1-padded for decode rows. + SWA_TOTAL_WIDTH: tl.constexpr = SWA_INDEX_WIDTH + IMAGE_WIDTH + + if token_idx < NUM_DECODE_TOKENS: + for i in range(0, SWA_TOTAL_WIDTH, WINDOW_BLOCK_SIZE): + offset = i + tl.arange(0, WINDOW_BLOCK_SIZE) + mask = offset < SWA_TOTAL_WIDTH + values = tl.load( + decode_swa_indices_ptr + token_idx * decode_swa_stride + offset, + mask=offset < SWA_INDEX_WIDTH, + other=-1, + ) + values = _remap_flashinfer_index(values, swa_block_size, swa_block_span) + tl.store( + sparse_indices_ptr + token_idx * sparse_indices_stride + offset, + values, + mask=mask, + ) + + compressed_len = tl.zeros((), dtype=tl.int32) + for i in range(0, PADDED_TOP_K, TOPK_BLOCK_SIZE): + offset = i + tl.arange(0, TOPK_BLOCK_SIZE) + mask = offset < PADDED_TOP_K + values = tl.load( + decode_compressed_indices_ptr + + token_idx * decode_compressed_stride + + offset, + mask=offset < DECODE_COMPRESSED_TOPK, + other=-1, + ) + if DECODE_COMPRESSED_INDICES_ARE_LOCAL: + token_valid = tl.load(decode_is_valid_token_ptr + token_idx) + is_valid = values >= 0 + req_idx = tl.load(token_to_req_indices_ptr + token_idx) + block_indices = values // compressed_block_size + block_numbers = tl.load( + compressed_block_table_ptr + + req_idx * compressed_block_table_stride + + block_indices, + mask=mask & is_valid, + other=-1, + ) + block_offsets = values % compressed_block_size + values = block_numbers * compressed_block_size + block_offsets + values = tl.where(is_valid, values, -1) + compressed_len += tl.sum( + (is_valid & token_valid).to(tl.int32), axis=0 + ) + values = _remap_flashinfer_index( + values, compressed_block_size, compressed_block_span + ) + tl.store( + sparse_indices_ptr + + token_idx * sparse_indices_stride + + SWA_TOTAL_WIDTH + + offset, + values, + mask=mask, + ) + + if DECODE_COMPRESSED_TOPK == 0: + compressed_len = tl.zeros((), dtype=tl.int32) + elif not DECODE_COMPRESSED_INDICES_ARE_LOCAL: + if HAS_DECODE_COMPRESSED_LENS: + compressed_len = tl.load( + decode_compressed_topk_lens_ptr + token_idx + ) + else: + compressed_len = tl.full((), DECODE_COMPRESSED_TOPK, dtype=tl.int32) + + tl.store(sparse_topk_lens_ptr + token_idx, SWA_TOTAL_WIDTH + compressed_len) + return + + prefill_idx = token_idx - NUM_DECODE_TOKENS + req_idx = tl.load(token_to_req_indices_ptr + token_idx) + query_start = tl.load(query_start_loc_ptr + req_idx) + query_end = tl.load(query_start_loc_ptr + req_idx + 1) + query_len = query_end - query_start + seq_len = tl.load(seq_lens_ptr + req_idx) + start_pos = seq_len - query_len + token_idx_in_query = token_idx - query_start + pos = start_pos + token_idx_in_query + if IMAGE_WIDTH > 0: + # In-image bidirectional visibility: window starts up to + # max(left - (window - 1), 0) earlier and extends `right` past pos. + left = tl.load(left_visible_ptr + token_idx) + right = tl.load(right_visible_ptr + token_idx) + else: + left = 0 + right = 0 + left_add = tl.maximum(left - (WINDOW_SIZE - 1), 0) + swa_start_pos = tl.maximum(pos - (WINDOW_SIZE - 1) - left_add, 0) + swa_len = pos + right - swa_start_pos + 1 + topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K) - if token_idx < NUM_DECODE_TOKENS: for i in range(0, SWA_TOTAL_WIDTH, WINDOW_BLOCK_SIZE): offset = i + tl.arange(0, WINDOW_BLOCK_SIZE) mask = offset < SWA_TOTAL_WIDTH - values = tl.load( - decode_swa_indices_ptr + token_idx * decode_swa_stride + offset, - mask=offset < SWA_INDEX_WIDTH, + pos_offset = swa_start_pos + offset + block_indices = pos_offset // swa_block_size + block_numbers = tl.load( + swa_block_table_ptr + req_idx * swa_block_table_stride + block_indices, + mask=mask & (offset < swa_len), other=-1, ) - values = _remap_flashinfer_index(values, swa_block_size, swa_block_span) + block_offsets = pos_offset % swa_block_size + slot_ids = block_numbers * swa_block_size + block_offsets + slot_ids = tl.where(offset < swa_len, slot_ids, -1) + slot_ids = _remap_flashinfer_index(slot_ids, swa_block_size, swa_block_span) tl.store( sparse_indices_ptr + token_idx * sparse_indices_stride + offset, - values, + slot_ids, mask=mask, ) - compressed_len = tl.zeros((), dtype=tl.int32) for i in range(0, PADDED_TOP_K, TOPK_BLOCK_SIZE): offset = i + tl.arange(0, TOPK_BLOCK_SIZE) mask = offset < PADDED_TOP_K - values = tl.load( - decode_compressed_indices_ptr - + token_idx * decode_compressed_stride - + offset, - mask=offset < DECODE_COMPRESSED_TOPK, + local_idx = tl.load( + prefill_topk_indices_ptr + prefill_idx * prefill_topk_stride + offset, + mask=(offset < PREFILL_TOPK_STRIDE) & (offset < topk_len), other=-1, ) - if DECODE_COMPRESSED_INDICES_ARE_LOCAL: - token_valid = tl.load(decode_is_valid_token_ptr + token_idx) - is_valid = values >= 0 - req_idx = tl.load(token_to_req_indices_ptr + token_idx) - block_indices = values // compressed_block_size - block_numbers = tl.load( - compressed_block_table_ptr - + req_idx * compressed_block_table_stride - + block_indices, - mask=mask & is_valid, - other=-1, - ) - block_offsets = values % compressed_block_size - values = block_numbers * compressed_block_size + block_offsets - values = tl.where(is_valid, values, -1) - compressed_len += tl.sum((is_valid & token_valid).to(tl.int32), axis=0) - values = _remap_flashinfer_index( - values, compressed_block_size, compressed_block_span + is_valid = local_idx >= 0 + block_indices = local_idx // compressed_block_size + block_numbers = tl.load( + compressed_block_table_ptr + + req_idx * compressed_block_table_stride + + block_indices, + mask=mask & is_valid, + other=-1, + ) + block_offsets = local_idx % compressed_block_size + slot_ids = block_numbers * compressed_block_size + block_offsets + slot_ids = tl.where((offset < topk_len) & is_valid, slot_ids, -1) + slot_ids = _remap_flashinfer_index( + slot_ids, compressed_block_size, compressed_block_span ) tl.store( sparse_indices_ptr + token_idx * sparse_indices_stride + SWA_TOTAL_WIDTH + offset, - values, + slot_ids, mask=mask, ) - if DECODE_COMPRESSED_TOPK == 0: - compressed_len = tl.zeros((), dtype=tl.int32) - elif not DECODE_COMPRESSED_INDICES_ARE_LOCAL: - if HAS_DECODE_COMPRESSED_LENS: - compressed_len = tl.load(decode_compressed_topk_lens_ptr + token_idx) - else: - compressed_len = tl.full((), DECODE_COMPRESSED_TOPK, dtype=tl.int32) + tl.store(sparse_topk_lens_ptr + token_idx, SWA_TOTAL_WIDTH + topk_len) - tl.store(sparse_topk_lens_ptr + token_idx, SWA_TOTAL_WIDTH + compressed_len) - return - - prefill_idx = token_idx - NUM_DECODE_TOKENS - req_idx = tl.load(token_to_req_indices_ptr + token_idx) - query_start = tl.load(query_start_loc_ptr + req_idx) - query_end = tl.load(query_start_loc_ptr + req_idx + 1) - query_len = query_end - query_start - seq_len = tl.load(seq_lens_ptr + req_idx) - start_pos = seq_len - query_len - token_idx_in_query = token_idx - query_start - pos = start_pos + token_idx_in_query - if IMAGE_WIDTH > 0: - # In-image bidirectional visibility: window starts up to - # max(left - (window - 1), 0) earlier and extends `right` past pos. - left = tl.load(left_visible_ptr + token_idx) - right = tl.load(right_visible_ptr + token_idx) - else: - left = 0 - right = 0 - left_add = tl.maximum(left - (WINDOW_SIZE - 1), 0) - swa_start_pos = tl.maximum(pos - (WINDOW_SIZE - 1) - left_add, 0) - swa_len = pos + right - swa_start_pos + 1 - topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K) - - for i in range(0, SWA_TOTAL_WIDTH, WINDOW_BLOCK_SIZE): - offset = i + tl.arange(0, WINDOW_BLOCK_SIZE) - mask = offset < SWA_TOTAL_WIDTH - pos_offset = swa_start_pos + offset - block_indices = pos_offset // swa_block_size - block_numbers = tl.load( - swa_block_table_ptr + req_idx * swa_block_table_stride + block_indices, - mask=mask & (offset < swa_len), - other=-1, + def dispatch( # type: ignore[override] + self, + *, + swa_index_width: int, + image_width: int, + compress_ratio: int, + index_topk: int, + active_topk_width: int, + has_prefill: bool, + **compile_key_fields: Any, + ) -> CompileKey: + decode_compressed_topk = ( + active_topk_width + if compress_ratio == 128 + else index_topk + if compress_ratio == 4 + else 0 ) - block_offsets = pos_offset % swa_block_size - slot_ids = block_numbers * swa_block_size + block_offsets - slot_ids = tl.where(offset < swa_len, slot_ids, -1) - slot_ids = _remap_flashinfer_index(slot_ids, swa_block_size, swa_block_span) - tl.store( - sparse_indices_ptr + token_idx * sparse_indices_stride + offset, - slot_ids, - mask=mask, + topk_width = decode_compressed_topk if has_prefill else 0 + padded_top_k = max(topk_width, decode_compressed_topk) + padded_top_k = (padded_top_k + 3) // 4 * 4 + return self.CompileKey( + **compile_key_fields, + compress_ratio=compress_ratio, + top_k=topk_width, + decode_compressed_indices_are_local=compress_ratio == 4, + has_decode_compressed_lens=compress_ratio == 128, + swa_index_width=swa_index_width, + image_width=image_width, + padded_top_k=padded_top_k, + decode_compressed_topk=decode_compressed_topk, + window_block_size=next_power_of_2(max(swa_index_width, 1)), + topk_block_size=next_power_of_2(max(padded_top_k, 1)), ) - for i in range(0, PADDED_TOP_K, TOPK_BLOCK_SIZE): - offset = i + tl.arange(0, TOPK_BLOCK_SIZE) - mask = offset < PADDED_TOP_K - local_idx = tl.load( - prefill_topk_indices_ptr + prefill_idx * prefill_topk_stride + offset, - mask=(offset < PREFILL_TOPK_STRIDE) & (offset < topk_len), - other=-1, + def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]: + hf_config = vllm_config.model_config.hf_config + window_size = int(getattr(hf_config, "sliding_window", 128) or 128) + speculative_config = vllm_config.speculative_config + if speculative_config is not None and speculative_config.use_dspark(): + from vllm.v1.attention.backends.mla.compressor_utils import ( + get_dspark_swa_index_width, + ) + + swa_index_width: int | tuple[int, int] = ( + window_size, + get_dspark_swa_index_width( + window_size, + speculative_config.num_speculative_tokens, + ), + ) + else: + swa_index_width = window_size + max_image_tokens = ( + int(getattr(hf_config, "vision_max_n_token", 0) or 0) + if int(getattr(hf_config, "vision_n_layers", 0) or 0) > 0 + else 0 ) - is_valid = local_idx >= 0 - block_indices = local_idx // compressed_block_size - block_numbers = tl.load( - compressed_block_table_ptr - + req_idx * compressed_block_table_stride - + block_indices, - mask=mask & is_valid, - other=-1, + image_widths = (0, max_image_tokens) if max_image_tokens > 0 else 0 + index_topk = int(getattr(hf_config, "index_topk", 0) or 0) + compress_ratios = tuple( + sorted({max(1, int(ratio)) for ratio in hf_config.compress_ratios}) ) - block_offsets = local_idx % compressed_block_size - slot_ids = block_numbers * compressed_block_size + block_offsets - slot_ids = tl.where((offset < topk_len) & is_valid, slot_ids, -1) - slot_ids = _remap_flashinfer_index( - slot_ids, compressed_block_size, compressed_block_span + max_c128a_topk = ( + ((vllm_config.model_config.max_model_len + 127) // 128 + 127) // 128 * 128 ) - tl.store( - sparse_indices_ptr - + token_idx * sparse_indices_stride - + SWA_TOTAL_WIDTH - + offset, - slot_ids, - mask=mask, + active_topk_widths = WarmupIntRange( + 128, + max_c128a_topk + 1, + advance=lambda width: ( + min(width * 2, max_c128a_topk) + if width < max_c128a_topk + else max_c128a_topk + 1 + ), + ) + return self._trace_dispatch(self.dispatch)( + compress_ratio=compress_ratios, + index_topk=index_topk, + active_topk_width=active_topk_widths, + has_prefill=(False, True), + window_size=window_size, + swa_index_width=swa_index_width, + image_width=image_widths, ) - tl.store(sparse_topk_lens_ptr + token_idx, SWA_TOTAL_WIDTH + topk_len) + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + int32_ptr = TritonWarmupTensor(torch.int32) + swa_block_size = 64 + compressed_block_size = ( + swa_block_size + if compile_key.compress_ratio == 1 + else 256 // compile_key.compress_ratio + ) + num_warps = ( + 4 + if max(compile_key.window_block_size, compile_key.topk_block_size) >= 256 + else 1 + ) + return dict( + sparse_indices=TritonWarmupTensor( + torch.int32, + shape=( + 1, + compile_key.swa_index_width + + compile_key.image_width + + compile_key.padded_top_k, + ), + ), + sparse_topk_lens=int32_ptr, + decode_swa_indices=TritonWarmupTensor( + torch.int32, + shape=(1, compile_key.swa_index_width), + ), + decode_compressed_indices=int32_ptr, + decode_compressed_topk_lens=int32_ptr, + decode_is_valid_token=TritonWarmupTensor( + torch.bool + if compile_key.decode_compressed_indices_are_local + else torch.int32 + ), + prefill_topk_indices=int32_ptr, + prefill_left_visible=int32_ptr, + prefill_right_visible=int32_ptr, + query_start_loc=int32_ptr, + seq_lens=int32_ptr, + token_to_req_indices=int32_ptr, + swa_block_table=TritonWarmupTensor(torch.int32, shape=(1, 1)), + swa_block_size=swa_block_size, + swa_block_span=1, + compressed_block_table=TritonWarmupTensor(torch.int32, shape=(1, 1)), + compressed_block_size=compressed_block_size, + compressed_block_span=1, + num_decode_tokens=1, + window_size=compile_key.window_size, + swa_index_width=compile_key.swa_index_width, + image_width=compile_key.image_width, + compress_ratio=compile_key.compress_ratio, + topk=compile_key.top_k, + padded_topk=compile_key.padded_top_k, + prefill_topk_stride=1, + decode_compressed_topk=compile_key.decode_compressed_topk, + decode_compressed_indices_are_local=( + compile_key.decode_compressed_indices_are_local + ), + has_decode_compressed_lens=compile_key.has_decode_compressed_lens, + window_block_size=compile_key.window_block_size, + topk_block_size=compile_key.topk_block_size, + num_warps=num_warps, + ) + + @kernel_launcher + def __call__( + self, + sparse_indices: torch.Tensor, + sparse_topk_lens: torch.Tensor, + decode_swa_indices: torch.Tensor, + decode_compressed_indices: torch.Tensor, + decode_compressed_topk_lens: torch.Tensor, + decode_is_valid_token: torch.Tensor, + prefill_topk_indices: torch.Tensor, + prefill_left_visible: torch.Tensor, + prefill_right_visible: torch.Tensor, + query_start_loc: torch.Tensor, + seq_lens: torch.Tensor, + token_to_req_indices: torch.Tensor, + swa_block_table: torch.Tensor, + swa_block_size: int, + swa_block_span: int, + compressed_block_table: torch.Tensor, + compressed_block_size: int, + compressed_block_span: int, + *, + num_decode_tokens: int, + window_size: int, + swa_index_width: int, + image_width: int, + compress_ratio: int, + topk: int, + padded_topk: int, + prefill_topk_stride: int, + decode_compressed_topk: int, + decode_compressed_indices_are_local: bool, + has_decode_compressed_lens: bool, + window_block_size: int, + topk_block_size: int, + num_warps: int, + ) -> LaunchSpec: + return (sparse_indices.shape[0],), dict( + sparse_indices_stride=sparse_indices.stride(0), + decode_swa_stride=decode_swa_indices.stride(0), + decode_compressed_stride=decode_compressed_indices.stride(0), + prefill_topk_stride=prefill_topk_indices.stride(0), + left_visible_ptr=prefill_left_visible, + right_visible_ptr=prefill_right_visible, + swa_block_table_stride=swa_block_table.stride(0), + compressed_block_table_stride=compressed_block_table.stride(0), + NUM_DECODE_TOKENS=num_decode_tokens, + WINDOW_SIZE=window_size, + SWA_INDEX_WIDTH=swa_index_width, + IMAGE_WIDTH=image_width, + COMPRESS_RATIO=compress_ratio, + TOP_K=topk, + PADDED_TOP_K=padded_topk, + PREFILL_TOPK_STRIDE=prefill_topk_stride, + DECODE_COMPRESSED_TOPK=decode_compressed_topk, + DECODE_COMPRESSED_INDICES_ARE_LOCAL=decode_compressed_indices_are_local, + HAS_DECODE_COMPRESSED_LENS=has_decode_compressed_lens, + WINDOW_BLOCK_SIZE=window_block_size, + TOPK_BLOCK_SIZE=topk_block_size, + num_warps=num_warps, + ) + + +_BUILD_FLASHINFER_MIXED_SPARSE_INDICES_KERNEL = ( + BuildFlashinferMixedSparseIndicesKernel() +) diff --git a/vllm/models/deepseek_v4/common/ops/fused_compress_quant_cache.py b/vllm/models/deepseek_v4/common/ops/fused_compress_quant_cache.py index 4c9f464ef0..e0bc2a0428 100644 --- a/vllm/models/deepseek_v4/common/ops/fused_compress_quant_cache.py +++ b/vllm/models/deepseek_v4/common/ops/fused_compress_quant_cache.py @@ -19,13 +19,22 @@ even/odd halves, producing (N_QUANT_BLOCKS, MXFP4_BLOCK/2) packed nibbles and N_QUANT_BLOCKS ue8m0 bytes. """ +from dataclasses import dataclass from functools import lru_cache +from types import SimpleNamespace from typing import Any import torch +from vllm.model_executor.warmup.jit_warmup_triton_helper import ( + LaunchSpec, + TritonWarmupTensor, + VllmTritonJitKernel, + kernel_launcher, +) from vllm.platforms import current_platform from vllm.triton_utils import tl, triton +from vllm.utils.math_utils import round_up if current_platform.is_rocm(): from vllm.platforms.rocm import _ON_GFX950 @@ -68,14 +77,32 @@ def compress_norm_rope_store_triton( kernel = _fused_kv_compress_norm_rope_insert_sparse_attn num_warps = 4 kernel_kwargs = {"SANITIZE_CACHE_NANS": _ON_GFX950} - elif use_fp4_cache: - kernel = _fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn - num_warps = 1 - kernel_kwargs = {} else: - kernel = _fused_kv_compress_norm_rope_insert_indexer_attn - num_warps = 1 - kernel_kwargs = {} + _FUSED_KV_COMPRESS_NORM_ROPE_INSERT_INDEXER_TRITON_KERNEL( + state_cache=state_cache, + num_actual=num_actual, + token_to_req_indices=token_to_req_indices, + positions=positions, + slot_mapping=slot_mapping, + block_table=block_table, + block_size=block_size, + state_width=state_width, + cos_sin_cache=cos_sin_cache, + kv_cache=kv_cache, + k_cache_metadata=k_cache_metadata, + pdl_kwargs=pdl_kwargs, + head_dim=head_dim, + rope_head_dim=rope_head_dim, + compress_ratio=compress_ratio, + overlap=overlap, + use_fp4_cache=use_fp4_cache, + rms_norm_weight=rms_norm_weight, + rms_norm_eps=rms_norm_eps, + quant_block=quant_block, + token_stride=token_stride, + scale_dim=scale_dim, + ) + return kernel[(num_actual,)]( # state cache @@ -673,7 +700,15 @@ def compress_norm_rope_store_two_stage_triton( # ============================================================================= # Indexer path (head=128, all FP8, single quant block) # ============================================================================= -@triton.jit +@triton.jit( + do_not_specialize=[ + "state_cache_stride0", + "state_cache_stride1", + "block_table_stride", + "rms_norm_eps", + "cos_sin_stride", + ] +) def _fused_kv_compress_norm_rope_insert_indexer_attn( # ── state cache (compressor internal state) ── state_cache_ptr, @@ -850,7 +885,15 @@ def _fused_kv_compress_norm_rope_insert_indexer_attn( # ============================================================================= # Indexer path (head=128, MXFP4: 2 nibbles/byte + ue8m0 per 32-elem block) # ============================================================================= -@triton.jit +@triton.jit( + do_not_specialize=[ + "state_cache_stride0", + "state_cache_stride1", + "block_table_stride", + "rms_norm_eps", + "cos_sin_stride", + ] +) def _fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn( # ── state cache (compressor internal state) ── state_cache_ptr, @@ -1038,3 +1081,232 @@ def _fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn( tl.store(val_ptr + tl.arange(0, TOKEN_STRIDE), packed_flat) tl.store(scale_ptr + tl.arange(0, SCALE_DIM), ue8m0) + + +class FusedKVCompressNormRopeInsertIndexerTritonKernel( + VllmTritonJitKernel["FusedKVCompressNormRopeInsertIndexerTritonKernel.CompileKey"] +): + @dataclass(frozen=True) + class CompileKey: + dtype: torch.dtype + use_fp4_cache: bool + head_size: int + triton_block_size: int + state_width: int + compress_ratio: int + overlap: bool + rope_head_dim: int + fp8_max: float + quant_block: int + token_stride: int + scale_dim: int + block_size: int + kv_cache_block_size: int + kv_block_stride: int + + # Canonical kernel for the launcher's arg_names mapping; the launched + # variant (fp8/mxfp4, same arg_names) is picked per call in __call__. + kernel = staticmethod(_fused_kv_compress_norm_rope_insert_indexer_attn) + + def dispatch( # type: ignore[override] + self, + *, + dtype: torch.dtype, + use_fp4_cache: bool, + head_dim: int, + rope_head_dim: int, + compress_ratio: int, + cache_block_size: int, + cache_alignment: int, + runtime_state_width: int | None = None, + runtime_quant_block: int | None = None, + runtime_token_stride: int | None = None, + runtime_scale_dim: int | None = None, + runtime_kv_block_stride: int | None = None, + ) -> CompileKey: + overlap = compress_ratio == 4 + default_quant_block = 32 if use_fp4_cache else 128 + quant_block = ( + runtime_quant_block + if runtime_quant_block is not None + else default_quant_block + ) + default_token_stride = head_dim // 2 if use_fp4_cache else head_dim + token_stride = ( + runtime_token_stride + if runtime_token_stride is not None + else default_token_stride + ) + default_scale_dim = head_dim // quant_block if use_fp4_cache else 4 + scale_dim = ( + runtime_scale_dim if runtime_scale_dim is not None else default_scale_dim + ) + raw_kv_cache_block_size = cache_block_size // compress_ratio + kv_cache_block_size = ( + raw_kv_cache_block_size if raw_kv_cache_block_size >= 1 else 1 + ) + state_block_size = ( + 4 + if compress_ratio == 4 + else 8 + if compress_ratio == 128 + else cache_block_size + ) + default_kv_block_stride = round_up( + kv_cache_block_size * (token_stride + scale_dim), + cache_alignment, + ) + kv_block_stride = ( + runtime_kv_block_stride + if runtime_kv_block_stride is not None + else default_kv_block_stride + ) + state_width = ( + runtime_state_width + if runtime_state_width is not None + else head_dim * (1 + overlap) + ) + return self.CompileKey( + dtype=dtype, + use_fp4_cache=use_fp4_cache, + head_size=head_dim, + triton_block_size=triton.next_power_of_2(head_dim), + state_width=state_width, + compress_ratio=compress_ratio, + overlap=overlap, + rope_head_dim=rope_head_dim, + fp8_max=448.0, + quant_block=quant_block, + token_stride=token_stride, + scale_dim=scale_dim, + block_size=state_block_size, + kv_cache_block_size=kv_cache_block_size, + kv_block_stride=kv_block_stride, + ) + + def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]: + hf_config = vllm_config.model_config.hf_config + if hf_config is None: + return [] + + cache_block_size = vllm_config.cache_config.block_size + from vllm.v1.attention.backends.mla.indexer import ( + dsa_indexer_uses_fp4, + ) + + use_fp4_cache = dsa_indexer_uses_fp4(vllm_config) + head_dim = int(getattr(hf_config, "index_head_dim", 0) or 0) + rope_head_dim = int(getattr(hf_config, "qk_rope_head_dim", 0) or 0) + if head_dim <= 0 or rope_head_dim <= 0 or cache_block_size <= 0: + return [] + + cache_dtype = vllm_config.cache_config.cache_dtype + cache_alignment = 576 if cache_dtype == "fp8_ds_mla" else 512 + return self._trace_dispatch(self.dispatch)( + dtype=vllm_config.model_config.dtype, + use_fp4_cache=use_fp4_cache, + head_dim=head_dim, + rope_head_dim=rope_head_dim, + compress_ratio=4, + cache_block_size=cache_block_size, + cache_alignment=cache_alignment, + ) + + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + fp32_ptr = TritonWarmupTensor(torch.float32, shape=(1, 1)) + int32_ptr = TritonWarmupTensor(torch.int32) + int64_ptr = TritonWarmupTensor(torch.int64) + # kv_cache supplies kv_cache_block_size (shape[1]) and KV_BLOCK_STRIDE + # (stride(0)); reproduce both so warmup and launch specialize identically. + kv_cache = TritonWarmupTensor( + torch.uint8, + shape=(1, compile_key.kv_cache_block_size, 1), + strides=(compile_key.kv_block_stride, 1, 1), + ) + return dict( + state_cache=fp32_ptr, + num_actual=1, + token_to_req_indices=int32_ptr, + positions=int64_ptr, + slot_mapping=int64_ptr, + block_table=int32_ptr, + block_size=compile_key.block_size, + state_width=compile_key.state_width, + cos_sin_cache=fp32_ptr, + kv_cache=kv_cache, + k_cache_metadata=SimpleNamespace(slot_mapping=int64_ptr), + pdl_kwargs={}, + head_dim=compile_key.head_size, + rope_head_dim=compile_key.rope_head_dim, + compress_ratio=compile_key.compress_ratio, + overlap=compile_key.overlap, + use_fp4_cache=compile_key.use_fp4_cache, + rms_norm_weight=TritonWarmupTensor( + compile_key.dtype, + shape=(compile_key.head_size,), + ), + rms_norm_eps=1e-6, + quant_block=compile_key.quant_block, + token_stride=compile_key.token_stride, + scale_dim=compile_key.scale_dim, + ) + + @kernel_launcher + def __call__( + self, + *, + state_cache: torch.Tensor, + num_actual: int, + token_to_req_indices: torch.Tensor, + positions: torch.Tensor, + slot_mapping: torch.Tensor, + block_table: torch.Tensor, + block_size: int, + state_width: int, + cos_sin_cache: torch.Tensor, + kv_cache: torch.Tensor, + k_cache_metadata: Any, + pdl_kwargs: dict, + head_dim: int, + rope_head_dim: int, + compress_ratio: int, + overlap: bool, + use_fp4_cache: bool, + rms_norm_weight: torch.Tensor, + rms_norm_eps: float, + quant_block: int, + token_stride: int, + scale_dim: int, + ) -> LaunchSpec: + return (num_actual,), dict( + kernel=( + _fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn + if use_fp4_cache + else _fused_kv_compress_norm_rope_insert_indexer_attn + ), + state_cache_stride0=state_cache.stride(0), + state_cache_stride1=state_cache.stride(1), + block_table_stride=block_table.stride(0), + cos_sin_stride=cos_sin_cache.stride(0), + k_cache_ptr=kv_cache, + kv_slot_mapping_ptr=k_cache_metadata.slot_mapping, + kv_cache_block_size=kv_cache.shape[1], + HEAD_SIZE=head_dim, + TRITON_BLOCK_SIZE=triton.next_power_of_2(head_dim), + STATE_WIDTH=state_width, + COMPRESS_RATIO=compress_ratio, + OVERLAP=overlap, + ROPE_HEAD_DIM=rope_head_dim, + FP8_MAX=448.0, + QUANT_BLOCK=quant_block, + TOKEN_STRIDE=token_stride, + SCALE_DIM=scale_dim, + KV_BLOCK_STRIDE=kv_cache.stride(0), + num_warps=1, + **pdl_kwargs, + ) + + +_FUSED_KV_COMPRESS_NORM_ROPE_INSERT_INDEXER_TRITON_KERNEL = ( + FusedKVCompressNormRopeInsertIndexerTritonKernel() +) diff --git a/vllm/models/deepseek_v4/common/ops/fused_indexer_q.py b/vllm/models/deepseek_v4/common/ops/fused_indexer_q.py index 9ae34de1a0..d5c7d669b2 100644 --- a/vllm/models/deepseek_v4/common/ops/fused_indexer_q.py +++ b/vllm/models/deepseek_v4/common/ops/fused_indexer_q.py @@ -1,8 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from dataclasses import dataclass +from typing import Any + import torch +from vllm.model_executor.warmup.jit_warmup_triton_helper import ( + LaunchSpec, + TritonWarmupTensor, + VllmTritonJitKernel, + kernel_launcher, +) from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from vllm.utils.import_utils import has_cutedsl @@ -67,229 +76,459 @@ def _quantize_mxfp4_pair(x_lo, x_hi): return packed, ue8m0 -@triton.jit -def _fused_indexer_q_rope_quant_kernel( - pos_ptr, - # Index Q RoPE - index_q_ptr, - index_q_stride0, - index_q_stride1, - index_q_cos_sin_ptr, - index_q_cos_sin_stride, - INDEX_Q_HALF_ROT_DIM: tl.constexpr, - # Index Q Quantize - index_q_fp8_ptr, - index_q_fp8_stride0, - index_q_fp8_stride1, - INDEX_Q_HEAD_DIM: tl.constexpr, - # Index weights - index_weights_ptr, - index_weights_stride, - index_weights_softmax_scale, - index_weights_head_scale, - index_weights_out_ptr, - index_weights_out_stride, - FP8_MAX: tl.constexpr = 448.0, - USE_FNUZ: tl.constexpr = False, - USE_EXPLICIT_FMA: tl.constexpr = False, +class FusedIndexerQRopeQuantTritonKernel( + VllmTritonJitKernel["FusedIndexerQRopeQuantTritonKernel.CompileKey"] ): - # Layout matches the unfused reference (DeepseekV4ScalingRotaryEmbedding - # + per_token_group_quant_fp8): GPT-J interleaved RoPE applied to the - # LAST rope_dim dims of each head; the leading [0, NOPE_DIM) is passed - # through unchanged. - INDEX_Q_ROT_DIM: tl.constexpr = 2 * INDEX_Q_HALF_ROT_DIM - INDEX_Q_NOPE_DIM: tl.constexpr = INDEX_Q_HEAD_DIM - INDEX_Q_ROT_DIM - tl.static_assert(INDEX_Q_NOPE_DIM >= 0) + @dataclass(frozen=True) + class CompileKey: + dtype: torch.dtype + num_heads: int + index_q_half_rot_dim: int + index_q_head_dim: int + fp8_max: float + use_fnuz: bool + use_explicit_fma: bool - tok_idx = tl.program_id(0) - head_idx = tl.program_id(1) - - pos = tl.load(pos_ptr + tok_idx) - cos, sin = _get_cos_sin( + @staticmethod + @triton.jit + def kernel( + pos_ptr, + # Index Q RoPE + index_q_ptr, + index_q_stride0, + index_q_stride1, index_q_cos_sin_ptr, index_q_cos_sin_stride, - pos, - INDEX_Q_HALF_ROT_DIM, - ) - half_offset = tl.arange(0, INDEX_Q_HALF_ROT_DIM) - base_ptr = index_q_ptr + tok_idx * index_q_stride0 + head_idx * index_q_stride1 + INDEX_Q_HALF_ROT_DIM: tl.constexpr, + # Index Q Quantize + index_q_fp8_ptr, + index_q_fp8_stride0, + index_q_fp8_stride1, + INDEX_Q_HEAD_DIM: tl.constexpr, + # Index weights + index_weights_ptr, + index_weights_stride, + index_weights_softmax_scale, + index_weights_head_scale, + index_weights_out_ptr, + index_weights_out_stride, + FP8_MAX: tl.constexpr = 448.0, + USE_FNUZ: tl.constexpr = False, + USE_EXPLICIT_FMA: tl.constexpr = False, + ): + # Layout matches the unfused reference (DeepseekV4ScalingRotaryEmbedding + # + per_token_group_quant_fp8): GPT-J interleaved RoPE applied to the + # LAST rope_dim dims of each head; the leading [0, NOPE_DIM) is passed + # through unchanged. + INDEX_Q_ROT_DIM: tl.constexpr = 2 * INDEX_Q_HALF_ROT_DIM + INDEX_Q_NOPE_DIM: tl.constexpr = INDEX_Q_HEAD_DIM - INDEX_Q_ROT_DIM + tl.static_assert(INDEX_Q_NOPE_DIM >= 0) - # Interleaved (GPT-J) RoPE on dims [NOPE_DIM, HEAD_DIM): - # even = q[NOPE_DIM + 2*i], odd = q[NOPE_DIM + 2*i + 1] - rot_base = base_ptr + INDEX_Q_NOPE_DIM - x_even = tl.load(rot_base + half_offset * 2).to(tl.float32) - x_odd = tl.load(rot_base + half_offset * 2 + 1).to(tl.float32) - if USE_EXPLICIT_FMA: - # Match HIP rotary_embedding contraction before bf16 materialization. - r_even = tl.fma(x_even, cos, -(x_odd * sin)) - r_odd = tl.fma(x_odd, cos, x_even * sin) - else: - r_even = x_even * cos - x_odd * sin - r_odd = x_odd * cos + x_even * sin + tok_idx = tl.program_id(0) + head_idx = tl.program_id(1) - # Match reference numerics: fp32 → bf16 → fp32 before the ue8m0 absmax. - # Same pattern as the K-side compressor kernel (fused_compress_quant_cache.py). - r_even = r_even.to(tl.bfloat16).to(tl.float32) - r_odd = r_odd.to(tl.bfloat16).to(tl.float32) - - amax = tl.maximum(tl.max(tl.abs(r_even)), tl.max(tl.abs(r_odd))) - if INDEX_Q_NOPE_DIM > 0: - nope_offset = tl.arange(0, INDEX_Q_NOPE_DIM) - x_nope = tl.load(base_ptr + nope_offset).to(tl.float32) - amax = tl.maximum(amax, tl.max(tl.abs(x_nope))) - index_q_scale = tl.div_rn(tl.maximum(amax, 1e-4), FP8_MAX) - index_q_scale = tl.math.exp2(tl.math.ceil(tl.math.log2(index_q_scale))) - - # Store quantized values to index_q_fp8. FNUZ (e4m3fnuz) on gfx942, OCP - # (e4m3fn) elsewhere -- matches the K cache. - fp8_dtype = tl.float8e4b8 if USE_FNUZ else tl.float8e4nv - fp8_base_ptr = ( - index_q_fp8_ptr + tok_idx * index_q_fp8_stride0 + head_idx * index_q_fp8_stride1 - ) - if INDEX_Q_NOPE_DIM > 0: - tl.store( - fp8_base_ptr + nope_offset, - tl.div_rn(x_nope, index_q_scale).to(fp8_dtype), + pos = tl.load(pos_ptr + tok_idx) + cos, sin = _get_cos_sin( + index_q_cos_sin_ptr, + index_q_cos_sin_stride, + pos, + INDEX_Q_HALF_ROT_DIM, ) - fp8_rot_base = fp8_base_ptr + INDEX_Q_NOPE_DIM - tl.store( - fp8_rot_base + half_offset * 2, - tl.div_rn(r_even, index_q_scale).to(fp8_dtype), - ) - tl.store( - fp8_rot_base + half_offset * 2 + 1, - tl.div_rn(r_odd, index_q_scale).to(fp8_dtype), - ) + half_offset = tl.arange(0, INDEX_Q_HALF_ROT_DIM) + base_ptr = index_q_ptr + tok_idx * index_q_stride0 + head_idx * index_q_stride1 - # FP8 weight-fold contract: - # index_weights_out = index_weights * q_scale * softmax_scale * head_scale - # The per-token-per-head q_scale (fp32) IS folded into the output weights - # here because FP8 Q is stored WITHOUT a companion scale tensor — the - # downstream fp8_fp4_mqa_logits/fp8_fp4_paged_mqa_logits kernels use `weights` to - # apply per-token Q scale inline. See the MXFP4 kernel below for the - # contrasting convention (scales live with the Q values, weights are NOT - # q-scaled). - index_weights = tl.load( - index_weights_ptr + tok_idx * index_weights_stride + head_idx - ) - index_weights = index_weights.to(tl.float32) - index_weights *= index_q_scale - index_weights *= index_weights_softmax_scale - index_weights *= index_weights_head_scale - tl.store( - index_weights_out_ptr + tok_idx * index_weights_out_stride + head_idx, - index_weights, - ) + # Interleaved (GPT-J) RoPE on dims [NOPE_DIM, HEAD_DIM): + # even = q[NOPE_DIM + 2*i], odd = q[NOPE_DIM + 2*i + 1] + rot_base = base_ptr + INDEX_Q_NOPE_DIM + x_even = tl.load(rot_base + half_offset * 2).to(tl.float32) + x_odd = tl.load(rot_base + half_offset * 2 + 1).to(tl.float32) + if USE_EXPLICIT_FMA: + # Match HIP rotary_embedding contraction before bf16 materialization. + r_even = tl.fma(x_even, cos, -(x_odd * sin)) + r_odd = tl.fma(x_odd, cos, x_even * sin) + else: + r_even = x_even * cos - x_odd * sin + r_odd = x_odd * cos + x_even * sin - -@triton.jit -def _fused_indexer_q_rope_mxfp4_kernel( - pos_ptr, - # Index Q RoPE input (fp/bf16) - index_q_ptr, - index_q_stride0, - index_q_stride1, - index_q_cos_sin_ptr, - index_q_cos_sin_stride, - INDEX_Q_HALF_ROT_DIM: tl.constexpr, - # MXFP4 Q outputs - index_q_mxfp4_ptr, # uint8, (T, H, HEAD_DIM // 2) - index_q_mxfp4_stride0, - index_q_mxfp4_stride1, - index_q_scale_ptr, # uint8 ue8m0, (T, H, HEAD_DIM // BLOCK) - index_q_scale_stride0, - index_q_scale_stride1, - INDEX_Q_HEAD_DIM: tl.constexpr, - MXFP4_BLOCK: tl.constexpr, - # Weights (NO per-token q_scale fold for MXFP4; per-block scales stay - # with the Q values in the output scale tensor). - index_weights_ptr, - index_weights_stride, - index_weights_softmax_scale, - index_weights_head_scale, - index_weights_out_ptr, - index_weights_out_stride, -): - INDEX_Q_ROT_DIM: tl.constexpr = 2 * INDEX_Q_HALF_ROT_DIM - INDEX_Q_NOPE_DIM: tl.constexpr = INDEX_Q_HEAD_DIM - INDEX_Q_ROT_DIM - NUM_NOPE_BLOCKS: tl.constexpr = INDEX_Q_NOPE_DIM // MXFP4_BLOCK - NUM_ROPE_BLOCKS: tl.constexpr = INDEX_Q_ROT_DIM // MXFP4_BLOCK - HALF_BLOCK: tl.constexpr = MXFP4_BLOCK // 2 - tl.static_assert(INDEX_Q_NOPE_DIM >= 0) - tl.static_assert(INDEX_Q_NOPE_DIM % MXFP4_BLOCK == 0) - tl.static_assert(INDEX_Q_ROT_DIM % MXFP4_BLOCK == 0) - tl.static_assert(MXFP4_BLOCK % 2 == 0) - - tok_idx = tl.program_id(0) - head_idx = tl.program_id(1) - - pos = tl.load(pos_ptr + tok_idx) - - q_base = index_q_ptr + tok_idx * index_q_stride0 + head_idx * index_q_stride1 - out_base = ( - index_q_mxfp4_ptr - + tok_idx * index_q_mxfp4_stride0 - + head_idx * index_q_mxfp4_stride1 - ) - scale_base = ( - index_q_scale_ptr - + tok_idx * index_q_scale_stride0 - + head_idx * index_q_scale_stride1 - ) - - half_off = tl.arange(0, HALF_BLOCK) - - # ---- NoPE blocks: direct load, pair as (even-index, odd-index) values ---- - for b in tl.static_range(NUM_NOPE_BLOCKS): - base = b * MXFP4_BLOCK - x_lo = tl.load(q_base + base + half_off * 2).to(tl.float32) - x_hi = tl.load(q_base + base + half_off * 2 + 1).to(tl.float32) - packed, ue8m0 = _quantize_mxfp4_pair(x_lo, x_hi) - tl.store(out_base + base // 2 + half_off, packed) - tl.store(scale_base + b, ue8m0) - - # ---- RoPE blocks: apply GPT-J interleaved RoPE to the block's 16 pairs, - # then quantize. Each block covers HALF_BLOCK (=16) cos/sin pairs. ---- - rot_q_base = q_base + INDEX_Q_NOPE_DIM - for b in tl.static_range(NUM_ROPE_BLOCKS): - pair_off = b * HALF_BLOCK + half_off # indices in [0, HALF_ROT_DIM) - cos_b = tl.load( - index_q_cos_sin_ptr + pos * index_q_cos_sin_stride + pair_off - ).to(tl.float32) - sin_b = tl.load( - index_q_cos_sin_ptr - + pos * index_q_cos_sin_stride - + pair_off - + INDEX_Q_HALF_ROT_DIM - ).to(tl.float32) - x_even = tl.load(rot_q_base + pair_off * 2).to(tl.float32) - x_odd = tl.load(rot_q_base + pair_off * 2 + 1).to(tl.float32) - r_even = x_even * cos_b - x_odd * sin_b - r_odd = x_odd * cos_b + x_even * sin_b - # bf16 roundtrip for parity with the FP8 kernel / reference numerics. + # Match reference numerics: fp32 → bf16 → fp32 before the ue8m0 absmax. + # Same pattern as the K-side compressor kernel (fused_compress_quant_cache.py). r_even = r_even.to(tl.bfloat16).to(tl.float32) r_odd = r_odd.to(tl.bfloat16).to(tl.float32) - packed, ue8m0 = _quantize_mxfp4_pair(r_even, r_odd) - rope_byte_off = (INDEX_Q_NOPE_DIM + b * MXFP4_BLOCK) // 2 - tl.store(out_base + rope_byte_off + half_off, packed) - tl.store(scale_base + NUM_NOPE_BLOCKS + b, ue8m0) - # MXFP4 weight-fold contract: - # index_weights_out = index_weights * softmax_scale * head_scale - # NOTE: q_scale is NOT folded here (contrast with the FP8 kernel above). - # MXFP4 Q emits a separate ue8m0 scale tensor of shape - # (T, H, HEAD_DIM // MXFP4_BLOCK) alongside the packed values, so each - # per-block scale is applied by the downstream MXFP4 logits kernel when - # dequantizing Q — there is no per-token scalar to fold into `weights`. - index_weights = tl.load( - index_weights_ptr + tok_idx * index_weights_stride + head_idx - ).to(tl.float32) - index_weights *= index_weights_softmax_scale - index_weights *= index_weights_head_scale - tl.store( - index_weights_out_ptr + tok_idx * index_weights_out_stride + head_idx, - index_weights, - ) + amax = tl.maximum(tl.max(tl.abs(r_even)), tl.max(tl.abs(r_odd))) + if INDEX_Q_NOPE_DIM > 0: + nope_offset = tl.arange(0, INDEX_Q_NOPE_DIM) + x_nope = tl.load(base_ptr + nope_offset).to(tl.float32) + amax = tl.maximum(amax, tl.max(tl.abs(x_nope))) + index_q_scale = tl.div_rn(tl.maximum(amax, 1e-4), FP8_MAX) + index_q_scale = tl.math.exp2(tl.math.ceil(tl.math.log2(index_q_scale))) + + # Store quantized values to index_q_fp8. FNUZ (e4m3fnuz) on gfx942, OCP + # (e4m3fn) elsewhere -- matches the K cache. + fp8_dtype = tl.float8e4b8 if USE_FNUZ else tl.float8e4nv + fp8_base_ptr = ( + index_q_fp8_ptr + + tok_idx * index_q_fp8_stride0 + + head_idx * index_q_fp8_stride1 + ) + if INDEX_Q_NOPE_DIM > 0: + tl.store( + fp8_base_ptr + nope_offset, + tl.div_rn(x_nope, index_q_scale).to(fp8_dtype), + ) + fp8_rot_base = fp8_base_ptr + INDEX_Q_NOPE_DIM + tl.store( + fp8_rot_base + half_offset * 2, + tl.div_rn(r_even, index_q_scale).to(fp8_dtype), + ) + tl.store( + fp8_rot_base + half_offset * 2 + 1, + tl.div_rn(r_odd, index_q_scale).to(fp8_dtype), + ) + + # FP8 weight-fold contract: + # index_weights_out = index_weights * q_scale * softmax_scale * head_scale + # The per-token-per-head q_scale (fp32) IS folded into the output weights + # here because FP8 Q is stored WITHOUT a companion scale tensor — the + # downstream fp8_fp4_mqa_logits/fp8_fp4_paged_mqa_logits kernels use + # `weights` to apply per-token Q scale inline. See the MXFP4 kernel below for + # contrasting convention (scales live with the Q values, weights are NOT + # q-scaled). + index_weights = tl.load( + index_weights_ptr + tok_idx * index_weights_stride + head_idx + ) + index_weights = index_weights.to(tl.float32) + index_weights *= index_q_scale + index_weights *= index_weights_softmax_scale + index_weights *= index_weights_head_scale + tl.store( + index_weights_out_ptr + tok_idx * index_weights_out_stride + head_idx, + index_weights, + ) + + def dispatch( # type: ignore[override] + self, + *, + dtype: torch.dtype, + num_heads: int, + head_dim: int, + rope_dim: int, + use_fnuz: bool, + ) -> CompileKey: + return self.CompileKey( + dtype=dtype, + num_heads=num_heads, + index_q_half_rot_dim=rope_dim // 2, + index_q_head_dim=head_dim, + fp8_max=224.0 if use_fnuz else 448.0, + use_fnuz=use_fnuz, + use_explicit_fma=current_platform.is_rocm(), + ) + + def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]: + hf_config = vllm_config.model_config.hf_config + num_heads = int(getattr(hf_config, "index_n_heads", 0) or 0) + head_dim = int(getattr(hf_config, "index_head_dim", 0) or 0) + rope_dim = int(getattr(hf_config, "qk_rope_head_dim", 0) or 0) + if num_heads <= 0 or head_dim <= 0 or rope_dim <= 0: + return [] + + return self._trace_dispatch(self.dispatch)( + dtype=vllm_config.model_config.dtype, + num_heads=num_heads, + head_dim=head_dim, + rope_dim=rope_dim, + use_fnuz=current_platform.fp8_dtype() == torch.float8_e4m3fnuz, + ) + + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + q_stride0 = compile_key.num_heads * compile_key.index_q_head_dim + return dict( + positions=TritonWarmupTensor(torch.int64), + index_q=TritonWarmupTensor( + compile_key.dtype, + shape=(1, compile_key.num_heads, compile_key.index_q_head_dim), + strides=(q_stride0, compile_key.index_q_head_dim, 1), + ), + index_q_cos_sin_cache=TritonWarmupTensor( + torch.float32, + shape=(1, compile_key.index_q_half_rot_dim * 2), + ), + index_weights=TritonWarmupTensor( + compile_key.dtype, + shape=(1, compile_key.num_heads), + ), + index_weights_softmax_scale=1.0, + index_weights_head_scale=1.0, + index_q_fp8=TritonWarmupTensor( + current_platform.fp8_dtype(), + shape=(1, compile_key.num_heads, compile_key.index_q_head_dim), + strides=(q_stride0, compile_key.index_q_head_dim, 1), + ), + index_weights_out=TritonWarmupTensor( + torch.float32, + shape=(1, compile_key.num_heads), + ), + fp8_max=compile_key.fp8_max, + use_fnuz=compile_key.use_fnuz, + ) + + @kernel_launcher + def __call__( + self, + positions: torch.Tensor, + index_q: torch.Tensor, + index_q_cos_sin_cache: torch.Tensor, + index_weights: torch.Tensor, + index_weights_softmax_scale: float, + index_weights_head_scale: float, + index_q_fp8: torch.Tensor, + index_weights_out: torch.Tensor, + *, + fp8_max: float, + use_fnuz: bool, + ) -> LaunchSpec: + num_tokens = positions.shape[0] + num_index_q_heads = index_q.shape[1] + return (num_tokens, num_index_q_heads), dict( + pos_ptr=positions, + index_q_stride0=index_q.stride(0), + index_q_stride1=index_q.stride(1), + index_q_cos_sin_ptr=index_q_cos_sin_cache, + index_q_cos_sin_stride=index_q_cos_sin_cache.stride(0), + INDEX_Q_HALF_ROT_DIM=index_q_cos_sin_cache.shape[-1] // 2, + index_q_fp8_stride0=index_q_fp8.stride(0), + index_q_fp8_stride1=index_q_fp8.stride(1), + INDEX_Q_HEAD_DIM=index_q.shape[2], + index_weights_stride=index_weights.stride(0), + index_weights_out_stride=index_weights_out.stride(0), + FP8_MAX=fp8_max, + USE_FNUZ=use_fnuz, + USE_EXPLICIT_FMA=current_platform.is_rocm(), + num_warps=1, + ) + + +class FusedIndexerQRopeMxFp4TritonKernel( + VllmTritonJitKernel["FusedIndexerQRopeMxFp4TritonKernel.CompileKey"] +): + @dataclass(frozen=True) + class CompileKey: + dtype: torch.dtype + num_heads: int + index_q_half_rot_dim: int + index_q_head_dim: int + mxfp4_block: int + + @staticmethod + @triton.jit + def kernel( + pos_ptr, + # Index Q RoPE input (fp/bf16) + index_q_ptr, + index_q_stride0, + index_q_stride1, + index_q_cos_sin_ptr, + index_q_cos_sin_stride, + INDEX_Q_HALF_ROT_DIM: tl.constexpr, + # MXFP4 Q outputs + index_q_mxfp4_ptr, # uint8, (T, H, HEAD_DIM // 2) + index_q_mxfp4_stride0, + index_q_mxfp4_stride1, + index_q_scale_ptr, # uint8 ue8m0, (T, H, HEAD_DIM // BLOCK) + index_q_scale_stride0, + index_q_scale_stride1, + INDEX_Q_HEAD_DIM: tl.constexpr, + MXFP4_BLOCK: tl.constexpr, + # Weights (NO per-token q_scale fold for MXFP4; per-block scales stay + # with the Q values in the output scale tensor). + index_weights_ptr, + index_weights_stride, + index_weights_softmax_scale, + index_weights_head_scale, + index_weights_out_ptr, + index_weights_out_stride, + ): + INDEX_Q_ROT_DIM: tl.constexpr = 2 * INDEX_Q_HALF_ROT_DIM + INDEX_Q_NOPE_DIM: tl.constexpr = INDEX_Q_HEAD_DIM - INDEX_Q_ROT_DIM + NUM_NOPE_BLOCKS: tl.constexpr = INDEX_Q_NOPE_DIM // MXFP4_BLOCK + NUM_ROPE_BLOCKS: tl.constexpr = INDEX_Q_ROT_DIM // MXFP4_BLOCK + HALF_BLOCK: tl.constexpr = MXFP4_BLOCK // 2 + tl.static_assert(INDEX_Q_NOPE_DIM >= 0) + tl.static_assert(INDEX_Q_NOPE_DIM % MXFP4_BLOCK == 0) + tl.static_assert(INDEX_Q_ROT_DIM % MXFP4_BLOCK == 0) + tl.static_assert(MXFP4_BLOCK % 2 == 0) + + tok_idx = tl.program_id(0) + head_idx = tl.program_id(1) + + pos = tl.load(pos_ptr + tok_idx) + + q_base = index_q_ptr + tok_idx * index_q_stride0 + head_idx * index_q_stride1 + out_base = ( + index_q_mxfp4_ptr + + tok_idx * index_q_mxfp4_stride0 + + head_idx * index_q_mxfp4_stride1 + ) + scale_base = ( + index_q_scale_ptr + + tok_idx * index_q_scale_stride0 + + head_idx * index_q_scale_stride1 + ) + + half_off = tl.arange(0, HALF_BLOCK) + + # ---- NoPE blocks: direct load, pair as (even-index, odd-index) values ---- + for b in tl.static_range(NUM_NOPE_BLOCKS): + base = b * MXFP4_BLOCK + x_lo = tl.load(q_base + base + half_off * 2).to(tl.float32) + x_hi = tl.load(q_base + base + half_off * 2 + 1).to(tl.float32) + packed, ue8m0 = _quantize_mxfp4_pair(x_lo, x_hi) + tl.store(out_base + base // 2 + half_off, packed) + tl.store(scale_base + b, ue8m0) + + # ---- RoPE blocks: apply GPT-J interleaved RoPE to the block's 16 pairs, + # then quantize. Each block covers HALF_BLOCK (=16) cos/sin pairs. ---- + rot_q_base = q_base + INDEX_Q_NOPE_DIM + for b in tl.static_range(NUM_ROPE_BLOCKS): + pair_off = b * HALF_BLOCK + half_off # indices in [0, HALF_ROT_DIM) + cos_b = tl.load( + index_q_cos_sin_ptr + pos * index_q_cos_sin_stride + pair_off + ).to(tl.float32) + sin_b = tl.load( + index_q_cos_sin_ptr + + pos * index_q_cos_sin_stride + + pair_off + + INDEX_Q_HALF_ROT_DIM + ).to(tl.float32) + x_even = tl.load(rot_q_base + pair_off * 2).to(tl.float32) + x_odd = tl.load(rot_q_base + pair_off * 2 + 1).to(tl.float32) + r_even = x_even * cos_b - x_odd * sin_b + r_odd = x_odd * cos_b + x_even * sin_b + # bf16 roundtrip for parity with the FP8 kernel / reference numerics. + r_even = r_even.to(tl.bfloat16).to(tl.float32) + r_odd = r_odd.to(tl.bfloat16).to(tl.float32) + packed, ue8m0 = _quantize_mxfp4_pair(r_even, r_odd) + rope_byte_off = (INDEX_Q_NOPE_DIM + b * MXFP4_BLOCK) // 2 + tl.store(out_base + rope_byte_off + half_off, packed) + tl.store(scale_base + NUM_NOPE_BLOCKS + b, ue8m0) + + # MXFP4 weight-fold contract: + # index_weights_out = index_weights * softmax_scale * head_scale + # NOTE: q_scale is NOT folded here (contrast with the FP8 kernel above). + # MXFP4 Q emits a separate ue8m0 scale tensor of shape + # (T, H, HEAD_DIM // MXFP4_BLOCK) alongside the packed values, so each + # per-block scale is applied by the downstream MXFP4 logits kernel when + # dequantizing Q — there is no per-token scalar to fold into `weights`. + index_weights = tl.load( + index_weights_ptr + tok_idx * index_weights_stride + head_idx + ).to(tl.float32) + index_weights *= index_weights_softmax_scale + index_weights *= index_weights_head_scale + tl.store( + index_weights_out_ptr + tok_idx * index_weights_out_stride + head_idx, + index_weights, + ) + + def dispatch( # type: ignore[override] + self, + *, + dtype: torch.dtype, + num_heads: int, + head_dim: int, + rope_dim: int, + ) -> CompileKey: + return self.CompileKey( + dtype=dtype, + num_heads=num_heads, + index_q_half_rot_dim=rope_dim // 2, + index_q_head_dim=head_dim, + mxfp4_block=MXFP4_BLOCK_SIZE, + ) + + def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]: + hf_config = vllm_config.model_config.hf_config + num_heads = int(getattr(hf_config, "index_n_heads", 0) or 0) + head_dim = int(getattr(hf_config, "index_head_dim", 0) or 0) + rope_dim = int(getattr(hf_config, "qk_rope_head_dim", 0) or 0) + if num_heads <= 0 or head_dim <= 0 or rope_dim <= 0: + return [] + + return self._trace_dispatch(self.dispatch)( + dtype=vllm_config.model_config.dtype, + num_heads=num_heads, + head_dim=head_dim, + rope_dim=rope_dim, + ) + + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + q_stride0 = compile_key.num_heads * compile_key.index_q_head_dim + packed_head_dim = compile_key.index_q_head_dim // 2 + scale_head_dim = compile_key.index_q_head_dim // compile_key.mxfp4_block + return dict( + positions=TritonWarmupTensor(torch.int64), + index_q=TritonWarmupTensor( + compile_key.dtype, + shape=(1, compile_key.num_heads, compile_key.index_q_head_dim), + strides=(q_stride0, compile_key.index_q_head_dim, 1), + ), + index_q_cos_sin_cache=TritonWarmupTensor( + torch.float32, + shape=(1, compile_key.index_q_half_rot_dim * 2), + ), + index_weights=TritonWarmupTensor( + compile_key.dtype, + shape=(1, compile_key.num_heads), + ), + index_weights_softmax_scale=1.0, + index_weights_head_scale=1.0, + index_q_packed=TritonWarmupTensor( + torch.uint8, + shape=(1, compile_key.num_heads, packed_head_dim), + ), + index_q_scale=TritonWarmupTensor( + torch.uint8, + shape=(1, compile_key.num_heads, scale_head_dim), + ), + index_weights_out=TritonWarmupTensor( + torch.float32, + shape=(1, compile_key.num_heads), + ), + ) + + @kernel_launcher + def __call__( + self, + positions: torch.Tensor, + index_q: torch.Tensor, + index_q_cos_sin_cache: torch.Tensor, + index_weights: torch.Tensor, + index_weights_softmax_scale: float, + index_weights_head_scale: float, + index_q_packed: torch.Tensor, + index_q_scale: torch.Tensor, + index_weights_out: torch.Tensor, + ) -> LaunchSpec: + num_tokens = positions.shape[0] + num_index_q_heads = index_q.shape[1] + return (num_tokens, num_index_q_heads), dict( + pos_ptr=positions, + index_q_stride0=index_q.stride(0), + index_q_stride1=index_q.stride(1), + index_q_cos_sin_ptr=index_q_cos_sin_cache, + index_q_cos_sin_stride=index_q_cos_sin_cache.stride(0), + INDEX_Q_HALF_ROT_DIM=index_q_cos_sin_cache.shape[-1] // 2, + index_q_mxfp4_ptr=index_q_packed, + index_q_mxfp4_stride0=index_q_packed.stride(0), + index_q_mxfp4_stride1=index_q_packed.stride(1), + index_q_scale_stride0=index_q_scale.stride(0), + index_q_scale_stride1=index_q_scale.stride(1), + INDEX_Q_HEAD_DIM=index_q.shape[2], + MXFP4_BLOCK=MXFP4_BLOCK_SIZE, + index_weights_stride=index_weights.stride(0), + index_weights_out_stride=index_weights_out.stride(0), + num_warps=1, + ) def fused_indexer_q_rope_quant( @@ -386,29 +625,16 @@ def fused_indexer_q_rope_quant( index_weights_out, ) else: - _fused_indexer_q_rope_mxfp4_kernel[(num_tokens, num_index_q_heads)]( + _FUSED_INDEXER_Q_ROPE_MXFP4_TRITON_KERNEL( positions, index_q, - index_q.stride(0), - index_q.stride(1), index_q_cos_sin_cache, - index_q_cos_sin_cache.stride(0), - index_q_cos_sin_cache.shape[-1] // 2, - index_q_packed, - index_q_packed.stride(0), - index_q_packed.stride(1), - index_q_scale, - index_q_scale.stride(0), - index_q_scale.stride(1), - index_q_head_dim, - MXFP4_BLOCK_SIZE, index_weights, - index_weights.stride(0), index_weights_softmax_scale, index_weights_head_scale, + index_q_packed, + index_q_scale, index_weights_out, - index_weights_out.stride(0), - num_warps=1, # TODO: Tune this ) # Values stay uint8 (2 E2M1 nibbles per byte). Scales are 4 ue8m0 @@ -453,27 +679,20 @@ def fused_indexer_q_rope_quant( index_weights_out, ) else: - _fused_indexer_q_rope_quant_kernel[(num_tokens, num_index_q_heads)]( + _FUSED_INDEXER_Q_ROPE_QUANT_TRITON_KERNEL( positions, index_q, - index_q.stride(0), - index_q.stride(1), index_q_cos_sin_cache, - index_q_cos_sin_cache.stride(0), - index_q_cos_sin_cache.shape[-1] // 2, - index_q_fp8, - index_q_fp8.stride(0), - index_q_fp8.stride(1), - index_q_head_dim, index_weights, - index_weights.stride(0), index_weights_softmax_scale, index_weights_head_scale, + index_q_fp8, index_weights_out, - index_weights_out.stride(0), - FP8_MAX=fp8_max, - USE_FNUZ=use_fnuz, - USE_EXPLICIT_FMA=current_platform.is_rocm(), - num_warps=1, # TODO: Tune this + fp8_max=fp8_max, + use_fnuz=use_fnuz, ) return index_q_fp8, index_weights_out + + +_FUSED_INDEXER_Q_ROPE_QUANT_TRITON_KERNEL = FusedIndexerQRopeQuantTritonKernel() +_FUSED_INDEXER_Q_ROPE_MXFP4_TRITON_KERNEL = FusedIndexerQRopeMxFp4TritonKernel() diff --git a/vllm/models/deepseek_v4/common/ops/fused_inv_rope_fp8_quant.py b/vllm/models/deepseek_v4/common/ops/fused_inv_rope_fp8_quant.py index aa8750580d..d716d34778 100644 --- a/vllm/models/deepseek_v4/common/ops/fused_inv_rope_fp8_quant.py +++ b/vllm/models/deepseek_v4/common/ops/fused_inv_rope_fp8_quant.py @@ -7,145 +7,327 @@ Output scale format is pre-transformed (MN-major TMA-aligned; FP32 on SM90, INT32-packed UE8M0 on SM100) so fp8_einsum skips transform_sf_into_required_layout. """ +from dataclasses import dataclass +from typing import Any + import torch +from vllm.model_executor.warmup.jit_warmup_triton_helper import ( + LaunchSpec, + TritonWarmupTensor, + VllmTritonJitKernel, + kernel_launcher, +) from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from vllm.utils.torch_utils import direct_register_custom_op -@triton.jit(do_not_specialize=["num_tokens"]) -def _fused_inv_rope_fp8_quant_per_head( - o_ptr, - positions_ptr, - cos_sin_cache_ptr, - fp8_ptr, - scale_ptr, - num_tokens, - heads_per_group: tl.constexpr, - o_stride_token, - o_stride_head, - cache_stride_pos, - fp8_stride_group, - fp8_stride_token, - scale_stride_group, - scale_stride_k, - fp8_max: tl.constexpr, - eps: tl.constexpr, - QUANT_GROUP_SIZE: tl.constexpr, - CHUNKS_PER_HEAD: tl.constexpr, - ROPE_START: tl.constexpr, - HALF_ROPE: tl.constexpr, - TMA_ALIGNED_SCALES: tl.constexpr, - USE_GDC: tl.constexpr, - launch_pdl: tl.constexpr, # triton metadata +class FusedInvRopeFP8QuantKernel( + VllmTritonJitKernel["FusedInvRopeFP8QuantKernel.CompileKey"] ): - # Cast every stride to int64 — without this, Python-int strides are - # inferred as int32 and `pid_token(int64) × stride(int32)` can lower to - # int32 arithmetic, wrapping past 2³¹ for large prefill batches → IMA. - pid_token = tl.program_id(0).to(tl.int64) - pid_gh = tl.program_id(1).to(tl.int64) - o_stride_token = o_stride_token.to(tl.int64) - o_stride_head = o_stride_head.to(tl.int64) - cache_stride_pos = cache_stride_pos.to(tl.int64) - fp8_stride_group = fp8_stride_group.to(tl.int64) - fp8_stride_token = fp8_stride_token.to(tl.int64) - scale_stride_group = scale_stride_group.to(tl.int64) - scale_stride_k = scale_stride_k.to(tl.int64) + @dataclass(frozen=True) + class CompileKey: + heads_per_group: int + fp8_max: float + quant_group_size: int + chunks_per_head: int + rope_start: int + half_rope: int + tma_aligned_scales: bool + launch_pdl: bool - g = pid_gh // heads_per_group - head_in_group = pid_gh % heads_per_group - global_head = pid_gh - qb_start = head_in_group * CHUNKS_PER_HEAD - if USE_GDC: - tl.extra.cuda.gdc_launch_dependents() - tl.extra.cuda.gdc_wait() - # Padding rows in the TMA-aligned scale buffer: fill with zero and skip quant. - if pid_token >= num_tokens: + @staticmethod + # scale_stride_k = align(num_tokens, 4) is only 4-aligned, so its Triton int + # class varies with batch size; not specialized so one warmup key covers all + # (int64-cast, scalar addressing only, so perf-neutral). + @triton.jit(do_not_specialize=["num_tokens", "scale_stride_k"]) + def kernel( + o_ptr, + positions_ptr, + cos_sin_cache_ptr, + fp8_ptr, + scale_ptr, + num_tokens, + heads_per_group: tl.constexpr, + o_stride_token, + o_stride_head, + cache_stride_pos, + fp8_stride_group, + fp8_stride_token, + scale_stride_group, + scale_stride_k, + fp8_max: tl.constexpr, + eps: tl.constexpr, + QUANT_GROUP_SIZE: tl.constexpr, + CHUNKS_PER_HEAD: tl.constexpr, + ROPE_START: tl.constexpr, + HALF_ROPE: tl.constexpr, + TMA_ALIGNED_SCALES: tl.constexpr, + launch_pdl: tl.constexpr, + ): + # Cast every stride to int64 — without this, Python-int strides are + # inferred as int32 and `pid_token(int64) × stride(int32)` can lower to + # int32 arithmetic, wrapping past 2³¹ for large prefill batches → IMA. + pid_token = tl.program_id(0).to(tl.int64) + pid_gh = tl.program_id(1).to(tl.int64) + o_stride_token = o_stride_token.to(tl.int64) + o_stride_head = o_stride_head.to(tl.int64) + cache_stride_pos = cache_stride_pos.to(tl.int64) + fp8_stride_group = fp8_stride_group.to(tl.int64) + fp8_stride_token = fp8_stride_token.to(tl.int64) + scale_stride_group = scale_stride_group.to(tl.int64) + scale_stride_k = scale_stride_k.to(tl.int64) + + g = pid_gh // heads_per_group + head_in_group = pid_gh % heads_per_group + global_head = pid_gh + qb_start = head_in_group * CHUNKS_PER_HEAD + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + tl.extra.cuda.gdc_wait() + # Padding rows in the TMA-aligned scale buffer: fill with zero and skip quant. + if pid_token >= num_tokens: + if TMA_ALIGNED_SCALES: + scale_addr = ( + scale_ptr + + g * scale_stride_group + + pid_token + + head_in_group * scale_stride_k + ) + tl.store(scale_addr, tl.zeros((), dtype=tl.int32)) + else: + block_offsets = tl.arange(0, CHUNKS_PER_HEAD) + qb_indices = qb_start + block_offsets + scale_addrs = ( + scale_ptr + + g * scale_stride_group + + pid_token + + qb_indices * scale_stride_k + ) + tl.store(scale_addrs, tl.zeros((CHUNKS_PER_HEAD,), dtype=tl.float32)) + return + + input_base = o_ptr + pid_token * o_stride_token + global_head * o_stride_head + + HEAD_DIM: tl.constexpr = CHUNKS_PER_HEAD * QUANT_GROUP_SIZE + offsets = tl.arange(0, HEAD_DIM) + x = tl.load(input_base + offsets).to(tl.float32) + + rope_abs_start: tl.constexpr = ( + CHUNKS_PER_HEAD - 1 + ) * QUANT_GROUP_SIZE + ROPE_START + pos = tl.load(positions_ptr + pid_token) + cache_base = cos_sin_cache_ptr + pos * cache_stride_pos + is_rope = offsets >= rope_abs_start + rope_local = offsets - rope_abs_start + + x_partner = tl.load(input_base + (offsets ^ 1), mask=is_rope, other=0.0).to( + tl.float32 + ) + cs_idx = tl.maximum(rope_local >> 1, 0) + cos_v = tl.load(cache_base + cs_idx, mask=is_rope, other=1.0) + sin_v = tl.load(cache_base + HALF_ROPE + cs_idx, mask=is_rope, other=0.0) + x_add = x * cos_v + x_partner * sin_v + x_sub = x * cos_v - x_partner * sin_v + is_even = (rope_local & 1) == 0 + rotated = tl.where(is_even, x_add, x_sub) + x = tl.where(is_rope, rotated, x) + + x_2d = tl.reshape(tl.abs(x), (CHUNKS_PER_HEAD, QUANT_GROUP_SIZE)) + block_absmax = tl.maximum(tl.max(x_2d, axis=1), eps) + scale_raw = block_absmax * (1.0 / fp8_max) + scales = tl.math.exp2(tl.ceil(tl.log2(scale_raw))) + + scales_exp = tl.reshape( + tl.broadcast_to( + tl.reshape(scales, (CHUNKS_PER_HEAD, 1)), + (CHUNKS_PER_HEAD, QUANT_GROUP_SIZE), + ), + (HEAD_DIM,), + ) + x_quant = tl.clamp(x / scales_exp, -fp8_max, fp8_max).to(tl.float8e4nv) + + fp8_base = ( + fp8_ptr + + g * fp8_stride_group + + pid_token * fp8_stride_token + + qb_start * QUANT_GROUP_SIZE + ) + tl.store(fp8_base + offsets, x_quant) + + block_offsets = tl.arange(0, CHUNKS_PER_HEAD) + qb_indices = qb_start + block_offsets if TMA_ALIGNED_SCALES: + scale_bits = scales.to(tl.int32, bitcast=True) + ue8m0_bytes = (scale_bits >> 23) & 0xFF + packed_val = tl.sum(ue8m0_bytes << (block_offsets * 8)) scale_addr = ( scale_ptr + g * scale_stride_group + pid_token + head_in_group * scale_stride_k ) - tl.store(scale_addr, tl.zeros((), dtype=tl.int32)) + tl.store(scale_addr, packed_val) else: - block_offsets = tl.arange(0, CHUNKS_PER_HEAD) - qb_indices = qb_start + block_offsets scale_addrs = ( scale_ptr + g * scale_stride_group + pid_token + qb_indices * scale_stride_k ) - tl.store(scale_addrs, tl.zeros((CHUNKS_PER_HEAD,), dtype=tl.float32)) - return + tl.store(scale_addrs, scales) - input_base = o_ptr + pid_token * o_stride_token + global_head * o_stride_head - - HEAD_DIM: tl.constexpr = CHUNKS_PER_HEAD * QUANT_GROUP_SIZE - offsets = tl.arange(0, HEAD_DIM) - x = tl.load(input_base + offsets).to(tl.float32) - - rope_abs_start: tl.constexpr = (CHUNKS_PER_HEAD - 1) * QUANT_GROUP_SIZE + ROPE_START - pos = tl.load(positions_ptr + pid_token) - cache_base = cos_sin_cache_ptr + pos * cache_stride_pos - is_rope = offsets >= rope_abs_start - rope_local = offsets - rope_abs_start - - x_partner = tl.load(input_base + (offsets ^ 1), mask=is_rope, other=0.0).to( - tl.float32 - ) - cs_idx = tl.maximum(rope_local >> 1, 0) - cos_v = tl.load(cache_base + cs_idx, mask=is_rope, other=1.0) - sin_v = tl.load(cache_base + HALF_ROPE + cs_idx, mask=is_rope, other=0.0) - x_add = x * cos_v + x_partner * sin_v - x_sub = x * cos_v - x_partner * sin_v - is_even = (rope_local & 1) == 0 - rotated = tl.where(is_even, x_add, x_sub) - x = tl.where(is_rope, rotated, x) - - x_2d = tl.reshape(tl.abs(x), (CHUNKS_PER_HEAD, QUANT_GROUP_SIZE)) - block_absmax = tl.maximum(tl.max(x_2d, axis=1), eps) - scale_raw = block_absmax * (1.0 / fp8_max) - scales = tl.math.exp2(tl.ceil(tl.log2(scale_raw))) - - scales_exp = tl.reshape( - tl.broadcast_to( - tl.reshape(scales, (CHUNKS_PER_HEAD, 1)), - (CHUNKS_PER_HEAD, QUANT_GROUP_SIZE), - ), - (HEAD_DIM,), - ) - x_quant = tl.clamp(x / scales_exp, -fp8_max, fp8_max).to(tl.float8e4nv) - - fp8_base = ( - fp8_ptr - + g * fp8_stride_group - + pid_token * fp8_stride_token - + qb_start * QUANT_GROUP_SIZE - ) - tl.store(fp8_base + offsets, x_quant) - - block_offsets = tl.arange(0, CHUNKS_PER_HEAD) - qb_indices = qb_start + block_offsets - if TMA_ALIGNED_SCALES: - scale_bits = scales.to(tl.int32, bitcast=True) - ue8m0_bytes = (scale_bits >> 23) & 0xFF - packed_val = tl.sum(ue8m0_bytes << (block_offsets * 8)) - scale_addr = ( - scale_ptr - + g * scale_stride_group - + pid_token - + head_in_group * scale_stride_k + def dispatch( # type: ignore[override] + self, + *, + heads_per_group: int, + head_dim: int, + nope_dim: int, + rope_dim: int, + quant_group_size: int, + runtime_fp8_max: float | None = None, + runtime_chunks_per_head: int | None = None, + runtime_rope_start: int | None = None, + runtime_half_rope: int | None = None, + **compile_key_fields: bool, + ) -> CompileKey: + fp8_max = ( + runtime_fp8_max + if runtime_fp8_max is not None + else torch.finfo(torch.float8_e4m3fn).max ) - tl.store(scale_addr, packed_val) - else: - scale_addrs = ( - scale_ptr + g * scale_stride_group + pid_token + qb_indices * scale_stride_k + chunks_per_head = ( + runtime_chunks_per_head + if runtime_chunks_per_head is not None + else head_dim // quant_group_size + ) + rope_start = ( + runtime_rope_start + if runtime_rope_start is not None + else nope_dim % quant_group_size + ) + half_rope = ( + runtime_half_rope if runtime_half_rope is not None else rope_dim // 2 + ) + return self.CompileKey( + **compile_key_fields, + heads_per_group=heads_per_group, + fp8_max=fp8_max, + quant_group_size=quant_group_size, + chunks_per_head=chunks_per_head, + rope_start=rope_start, + half_rope=half_rope, + ) + + def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]: + hf_config = vllm_config.model_config.hf_config + if hf_config is None: + return [] + + tp_size = vllm_config.parallel_config.tensor_parallel_size + num_heads = int(getattr(hf_config, "num_attention_heads", 0) or 0) + num_groups = int(getattr(hf_config, "o_groups", 0) or 0) + if num_heads <= 0 or num_groups <= 0: + return [] + + local_heads = num_heads // tp_size + local_groups = num_groups // tp_size + if local_groups <= 0: + return [] + + capability = current_platform.get_device_capability() + if capability is None: + return [] + + head_dim = int(getattr(hf_config, "head_dim", 512) or 512) + rope_dim = int(getattr(hf_config, "qk_rope_head_dim", 64) or 64) + return self._trace_dispatch(self.dispatch)( + heads_per_group=local_heads // local_groups, + head_dim=head_dim, + nope_dim=head_dim - rope_dim, + rope_dim=rope_dim, + quant_group_size=128, + tma_aligned_scales=capability.major >= 10, + launch_pdl=(False, True), + ) + + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + head_dim = compile_key.chunks_per_head * compile_key.quant_group_size + fp8_dim = compile_key.heads_per_group * head_dim + scale_dtype = torch.int32 if compile_key.tma_aligned_scales else torch.float32 + return dict( + o=TritonWarmupTensor( + torch.bfloat16, + shape=(1, compile_key.heads_per_group, head_dim), + ), + positions=TritonWarmupTensor(torch.int64), + cos_sin_cache=TritonWarmupTensor( + torch.float32, + shape=(1, compile_key.half_rope * 2), + ), + fp8_buf=TritonWarmupTensor( + torch.float8_e4m3fn, + shape=(compile_key.heads_per_group, 1, fp8_dim), + strides=(fp8_dim, fp8_dim, 1), + ), + scale_buf=TritonWarmupTensor( + scale_dtype, + shape=(compile_key.heads_per_group, 1, 16), + strides=(16, 16, 16), + ), + num_tokens=1, + heads_per_group=compile_key.heads_per_group, + quant_group_size=compile_key.quant_group_size, + chunks_per_head=compile_key.chunks_per_head, + rope_start=compile_key.rope_start, + half_rope=compile_key.half_rope, + tma_aligned_scales=compile_key.tma_aligned_scales, + fp8_max=compile_key.fp8_max, + launch_pdl=compile_key.launch_pdl, + grid=(1, compile_key.heads_per_group), + ) + + @kernel_launcher + def __call__( + self, + o: torch.Tensor, + positions: torch.Tensor, + cos_sin_cache: torch.Tensor, + fp8_buf: torch.Tensor, + scale_buf: torch.Tensor, + num_tokens: int, + *, + heads_per_group: int, + quant_group_size: int, + chunks_per_head: int, + rope_start: int, + half_rope: int, + tma_aligned_scales: bool, + fp8_max: float, + launch_pdl: bool, + grid: tuple[int, int], + ) -> LaunchSpec: + return grid, dict( + fp8_ptr=fp8_buf, + scale_ptr=scale_buf, + o_stride_token=o.stride(0), + o_stride_head=o.stride(1), + cache_stride_pos=cos_sin_cache.stride(0), + fp8_stride_group=fp8_buf.stride(0), + fp8_stride_token=fp8_buf.stride(1), + scale_stride_group=scale_buf.stride(0), + scale_stride_k=scale_buf.stride(2), + eps=1e-10, + QUANT_GROUP_SIZE=quant_group_size, + CHUNKS_PER_HEAD=chunks_per_head, + ROPE_START=rope_start, + HALF_ROPE=half_rope, + TMA_ALIGNED_SCALES=tma_aligned_scales, + launch_pdl=launch_pdl, + num_stages=1, + num_warps=1, ) - tl.store(scale_addrs, scales) def fused_inv_rope_fp8_quant( @@ -256,8 +438,8 @@ def _fused_inv_rope_fp8_quant_kernel_impl( (scale_inner * tma_aligned_T, 1, tma_aligned_T), ) grid = (tma_aligned_T, n_groups * heads_per_group) - use_gdc = current_platform.is_arch_support_pdl() - _fused_inv_rope_fp8_quant_per_head[grid]( + launch_pdl = current_platform.is_arch_support_pdl() + _FUSED_INV_ROPE_FP8_QUANT_KERNEL( o, positions, cos_sin_cache, @@ -265,24 +447,14 @@ def _fused_inv_rope_fp8_quant_kernel_impl( scale_buf, num_tokens, heads_per_group=heads_per_group, - o_stride_token=o.stride(0), - o_stride_head=o.stride(1), - cache_stride_pos=cos_sin_cache.stride(0), - fp8_stride_group=fp8_buf.stride(0), - fp8_stride_token=fp8_buf.stride(1), - scale_stride_group=scale_buf.stride(0), - scale_stride_k=scale_buf.stride(2), + quant_group_size=quant_group_size, + chunks_per_head=chunks_per_head, + rope_start=rope_start, + half_rope=half_rope, + tma_aligned_scales=tma_aligned_scales, fp8_max=fp8_max, - eps=1e-10, - QUANT_GROUP_SIZE=quant_group_size, - CHUNKS_PER_HEAD=chunks_per_head, - ROPE_START=rope_start, - HALF_ROPE=half_rope, - TMA_ALIGNED_SCALES=tma_aligned_scales, - USE_GDC=use_gdc, - launch_pdl=use_gdc, - num_stages=1, - num_warps=1, + launch_pdl=launch_pdl, + grid=grid, ) return fp8_buf, scale_buf @@ -326,3 +498,6 @@ direct_register_custom_op( op_func=_fused_inv_rope_fp8_quant_kernel_impl, fake_impl=_fused_inv_rope_fp8_quant_kernel_fake, ) + + +_FUSED_INV_ROPE_FP8_QUANT_KERNEL = FusedInvRopeFP8QuantKernel() diff --git a/vllm/models/deepseek_v4/common/ops/fused_mtp_input_rmsnorm.py b/vllm/models/deepseek_v4/common/ops/fused_mtp_input_rmsnorm.py index 874b4f6079..c71043f1ae 100644 --- a/vllm/models/deepseek_v4/common/ops/fused_mtp_input_rmsnorm.py +++ b/vllm/models/deepseek_v4/common/ops/fused_mtp_input_rmsnorm.py @@ -17,9 +17,19 @@ A single grid (T, hc_mult+1) drives both norms: task 0 is enorm on inputs_embeds[token, :], task k+1 is hnorm on previous_hidden_states[token, k, :]. """ +from dataclasses import dataclass +from typing import Any + import torch +from vllm.model_executor.warmup.jit_warmup_triton_helper import ( + LaunchSpec, + TritonWarmupTensor, + VllmTritonJitKernel, + kernel_launcher, +) from vllm.triton_utils import tl, triton +from vllm.utils.math_utils import next_power_of_2 @triton.jit @@ -40,164 +50,291 @@ def _rmsnorm_row( tl.store(out_row_ptr + block, y.to(out_row_ptr.dtype.element_ty), mask=mask) -@triton.jit -def _fused_mtp_input_rmsnorm_kernel( - inputs_embeds_ptr, - positions_ptr, - prev_hidden_ptr, - enorm_weight_ptr, - hnorm_weight_ptr, - enorm_out_ptr, - hnorm_out_ptr, - eps, - HIDDEN: tl.constexpr, - HC_MULT: tl.constexpr, - BLOCK_SIZE: tl.constexpr, +class FusedMTPInputRMSNormKernel( + VllmTritonJitKernel["FusedMTPInputRMSNormKernel.CompileKey"] ): - # int64 token index so per-token offsets don't overflow int32 at - # large num_tokens (matches the convention in fused_q_kv_rmsnorm). - token_idx = tl.program_id(0).to(tl.int64) - pid_task = tl.program_id(1) + @dataclass(frozen=True) + class CompileKey: + dtype: torch.dtype + hidden: int + hc_mult: int + block_size: int + eps: float - block = tl.arange(0, BLOCK_SIZE) - mask = block < HIDDEN + @staticmethod + @triton.jit + def kernel( + inputs_embeds_ptr, + positions_ptr, + prev_hidden_ptr, + enorm_weight_ptr, + hnorm_weight_ptr, + enorm_out_ptr, + hnorm_out_ptr, + eps, + HIDDEN: tl.constexpr, + HC_MULT: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + ): + # int64 token index so per-token offsets don't overflow int32 at + # large num_tokens (matches the convention in fused_q_kv_rmsnorm). + token_idx = tl.program_id(0).to(tl.int64) + pid_task = tl.program_id(1) - if pid_task == 0: - # enorm path: load inputs_embeds[token, :] then zero-mask at pos==0. - # Math is preserved: pos==0 → x=0 → variance=0 → RMSNorm output is 0 - # regardless of weight, matching torch.where(pos==0, 0, x) + RMSNorm. - pos = tl.load(positions_ptr + token_idx) - keep = pos != 0 - x = tl.load( - inputs_embeds_ptr + token_idx * HIDDEN + block, mask=mask, other=0.0 + block = tl.arange(0, BLOCK_SIZE) + mask = block < HIDDEN + + if pid_task == 0: + # enorm path: load inputs_embeds[token, :] then zero-mask at pos==0. + # Math is preserved: pos==0 → x=0 → variance=0 → RMSNorm output is 0 + # regardless of weight, matching torch.where(pos==0, 0, x) + RMSNorm. + pos = tl.load(positions_ptr + token_idx) + keep = pos != 0 + x = tl.load( + inputs_embeds_ptr + token_idx * HIDDEN + block, mask=mask, other=0.0 + ) + x = tl.where(keep, x, 0.0) + _rmsnorm_row( + x, + enorm_weight_ptr, + enorm_out_ptr + token_idx * HIDDEN, + block, + mask, + eps, + HIDDEN, + ) + else: + # hnorm path: load prev_hidden[token, slot, :]. + slot = pid_task - 1 + row_offset = (token_idx * HC_MULT + slot) * HIDDEN + x = tl.load(prev_hidden_ptr + row_offset + block, mask=mask, other=0.0) + _rmsnorm_row( + x, + hnorm_weight_ptr, + hnorm_out_ptr + row_offset, + block, + mask, + eps, + HIDDEN, + ) + + def dispatch( # type: ignore[override] + self, + *, + dtype: torch.dtype, + hidden: int, + hc_mult: int, + eps: float, + ) -> CompileKey: + return self.CompileKey( + dtype=dtype, + hidden=hidden, + hc_mult=hc_mult, + block_size=next_power_of_2(hidden), + eps=eps, ) - x = tl.where(keep, x, 0.0) - _rmsnorm_row( - x, - enorm_weight_ptr, - enorm_out_ptr + token_idx * HIDDEN, - block, - mask, - eps, - HIDDEN, + + def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]: + hf_config = vllm_config.model_config.hf_config + if int(getattr(hf_config, "num_nextn_predict_layers", 0) or 0) <= 0: + return [] + + return self._trace_dispatch(self.dispatch)( + dtype=vllm_config.model_config.dtype, + hidden=int(hf_config.hidden_size), + hc_mult=int(hf_config.hc_mult), + eps=float(hf_config.rms_norm_eps), ) - else: - # hnorm path: load prev_hidden[token, slot, :]. - slot = pid_task - 1 - row_offset = (token_idx * HC_MULT + slot) * HIDDEN - x = tl.load(prev_hidden_ptr + row_offset + block, mask=mask, other=0.0) - _rmsnorm_row( - x, - hnorm_weight_ptr, - hnorm_out_ptr + row_offset, - block, - mask, - eps, - HIDDEN, + + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + return dict( + inputs_embeds=TritonWarmupTensor( + compile_key.dtype, + shape=(1, compile_key.hidden), + ), + positions=TritonWarmupTensor(torch.int64), + previous_hidden_states=TritonWarmupTensor( + compile_key.dtype, + shape=(1, compile_key.hc_mult, compile_key.hidden), + ), + enorm_weight=TritonWarmupTensor( + compile_key.dtype, + shape=(compile_key.hidden,), + ), + hnorm_weight=TritonWarmupTensor( + compile_key.dtype, + shape=(compile_key.hidden,), + ), + _outputs=( + TritonWarmupTensor( + compile_key.dtype, + shape=(1, compile_key.hidden), + ), + TritonWarmupTensor( + compile_key.dtype, + shape=(1, compile_key.hc_mult, compile_key.hidden), + ), + ), + eps=compile_key.eps, + hc_mult=compile_key.hc_mult, + ) + + @kernel_launcher + def __call__( + self, + inputs_embeds: torch.Tensor, + positions: torch.Tensor, + previous_hidden_states: torch.Tensor, + enorm_weight: torch.Tensor, + hnorm_weight: torch.Tensor, + eps: float, + hc_mult: int, + *, + _outputs: tuple[Any, Any] | None = None, + ) -> LaunchSpec: + num_tokens, hidden = inputs_embeds.shape + if _outputs is None: + assert inputs_embeds.ndim == 2 + assert previous_hidden_states.ndim == 3 + assert previous_hidden_states.shape[1] == hc_mult + assert inputs_embeds.shape[0] == previous_hidden_states.shape[0], ( + "token dim mismatch" + ) + assert ( + hidden + == previous_hidden_states.shape[2] + == enorm_weight.shape[0] + == hnorm_weight.shape[0] + ) + assert ( + inputs_embeds.is_contiguous() and previous_hidden_states.is_contiguous() + ) + assert enorm_weight.is_contiguous() and hnorm_weight.is_contiguous() + _outputs = ( + torch.empty_like(inputs_embeds), + torch.empty_like(previous_hidden_states), + ) + + enorm_out, hnorm_out = _outputs + grid = None if num_tokens == 0 else (num_tokens, hc_mult + 1) + return ( + grid, + dict( + prev_hidden_ptr=previous_hidden_states, + enorm_out_ptr=enorm_out, + hnorm_out_ptr=hnorm_out, + HIDDEN=hidden, + HC_MULT=hc_mult, + BLOCK_SIZE=next_power_of_2(hidden), + ), + _outputs, ) -@triton.jit -def _mtp_shared_head_rmsnorm_kernel( - x_ptr, - weight_ptr, - out_ptr, - eps, - HIDDEN: tl.constexpr, - BLOCK_SIZE: tl.constexpr, +class MTPSharedHeadRMSNormKernel( + VllmTritonJitKernel["MTPSharedHeadRMSNormKernel.CompileKey"] ): - token_idx = tl.program_id(0).to(tl.int64) - block = tl.arange(0, BLOCK_SIZE) - mask = block < HIDDEN - x = tl.load(x_ptr + token_idx * HIDDEN + block, mask=mask, other=0.0) - _rmsnorm_row( - x, + @dataclass(frozen=True) + class CompileKey: + dtype: torch.dtype + hidden: int + block_size: int + eps: float + + @staticmethod + @triton.jit + def kernel( + x_ptr, weight_ptr, - out_ptr + token_idx * HIDDEN, - block, - mask, + out_ptr, eps, - HIDDEN, - ) + HIDDEN: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + ): + token_idx = tl.program_id(0).to(tl.int64) + block = tl.arange(0, BLOCK_SIZE) + mask = block < HIDDEN + x = tl.load(x_ptr + token_idx * HIDDEN + block, mask=mask, other=0.0) + _rmsnorm_row( + x, + weight_ptr, + out_ptr + token_idx * HIDDEN, + block, + mask, + eps, + HIDDEN, + ) + + def dispatch( # type: ignore[override] + self, + *, + dtype: torch.dtype, + hidden: int, + eps: float, + ) -> CompileKey: + return self.CompileKey( + dtype=dtype, + hidden=hidden, + block_size=next_power_of_2(hidden), + eps=eps, + ) + + def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]: + hf_config = vllm_config.model_config.hf_config + if int(getattr(hf_config, "num_nextn_predict_layers", 0) or 0) <= 0: + return [] + + return self._trace_dispatch(self.dispatch)( + dtype=vllm_config.model_config.dtype, + hidden=int(hf_config.hidden_size), + eps=float(hf_config.rms_norm_eps), + ) + + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + return dict( + hidden_states=TritonWarmupTensor( + compile_key.dtype, + shape=(1, compile_key.hidden), + ), + weight=TritonWarmupTensor( + compile_key.dtype, + shape=(compile_key.hidden,), + ), + _output=TritonWarmupTensor( + compile_key.dtype, + shape=(1, compile_key.hidden), + ), + eps=compile_key.eps, + ) + + @kernel_launcher + def __call__( + self, + hidden_states: torch.Tensor, + weight: torch.Tensor, + eps: float, + *, + _output: Any | None = None, + ) -> LaunchSpec: + num_tokens, hidden = hidden_states.shape + if _output is None: + assert hidden_states.ndim == 2 + assert hidden_states.is_contiguous() + assert weight.is_contiguous() + _output = torch.empty_like(hidden_states) + + grid = None if num_tokens == 0 else (num_tokens,) + return ( + grid, + dict( + x_ptr=hidden_states, + out_ptr=_output, + HIDDEN=hidden, + BLOCK_SIZE=next_power_of_2(hidden), + ), + _output, + ) -def mtp_shared_head_rmsnorm( - hidden_states: torch.Tensor, - weight: torch.Tensor, - eps: float, -) -> torch.Tensor: - """RMSNorm for MTP's SharedHead.norm, on (T, H) bf16 input. - - Uses the same ``_rmsnorm_row`` body as ``fused_mtp_input_rmsnorm`` so the - MTP draft path runs one consistent RMSNorm implementation end to end. - """ - assert hidden_states.ndim == 2 - assert hidden_states.is_contiguous() - assert weight.is_contiguous() - num_tokens, hidden = hidden_states.shape - out = torch.empty_like(hidden_states) - if num_tokens == 0: - return out - block_size = triton.next_power_of_2(hidden) - _mtp_shared_head_rmsnorm_kernel[(num_tokens,)]( - hidden_states, - weight, - out, - eps, - HIDDEN=hidden, - BLOCK_SIZE=block_size, - ) - return out - - -def fused_mtp_input_rmsnorm( - inputs_embeds: torch.Tensor, - positions: torch.Tensor, - previous_hidden_states: torch.Tensor, - enorm_weight: torch.Tensor, - hnorm_weight: torch.Tensor, - eps: float, - hc_mult: int, -) -> tuple[torch.Tensor, torch.Tensor]: - """Returns (enorm_out, hnorm_out). - - enorm_out has the same shape as inputs_embeds (2D, [T, H]). - hnorm_out has the same shape as previous_hidden_states (3D, [T, hc_mult, H]). - previous_hidden_states must already be reshaped to 3D. - """ - assert inputs_embeds.ndim == 2 - assert previous_hidden_states.ndim == 3 - assert previous_hidden_states.shape[1] == hc_mult - assert inputs_embeds.shape[0] == previous_hidden_states.shape[0], ( - "token dim mismatch" - ) - assert ( - inputs_embeds.shape[1] - == previous_hidden_states.shape[2] - == enorm_weight.shape[0] - == hnorm_weight.shape[0] - ) - assert inputs_embeds.is_contiguous() and previous_hidden_states.is_contiguous() - assert enorm_weight.is_contiguous() and hnorm_weight.is_contiguous() - - num_tokens, hidden = inputs_embeds.shape - enorm_out = torch.empty_like(inputs_embeds) - hnorm_out = torch.empty_like(previous_hidden_states) - if num_tokens == 0: - return enorm_out, hnorm_out - - block_size = triton.next_power_of_2(hidden) - _fused_mtp_input_rmsnorm_kernel[(num_tokens, hc_mult + 1)]( - inputs_embeds, - positions, - previous_hidden_states, - enorm_weight, - hnorm_weight, - enorm_out, - hnorm_out, - eps, - HIDDEN=hidden, - HC_MULT=hc_mult, - BLOCK_SIZE=block_size, - ) - return enorm_out, hnorm_out +_FUSED_MTP_INPUT_RMSNORM_KERNEL = FusedMTPInputRMSNormKernel() +_MTP_SHARED_HEAD_RMSNORM_KERNEL = MTPSharedHeadRMSNormKernel() diff --git a/vllm/models/deepseek_v4/common/ops/save_partial_states.py b/vllm/models/deepseek_v4/common/ops/save_partial_states.py index e3d7d38f45..0537feab29 100644 --- a/vllm/models/deepseek_v4/common/ops/save_partial_states.py +++ b/vllm/models/deepseek_v4/common/ops/save_partial_states.py @@ -1,101 +1,193 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from dataclasses import dataclass +from typing import Any + import torch +from vllm.model_executor.warmup.jit_warmup_triton_helper import ( + LaunchSpec, + TritonWarmupTensor, + VllmTritonJitKernel, + kernel_launcher, +) from vllm.triton_utils import tl, triton +from vllm.utils.math_utils import next_power_of_2 -def save_partial_states( - kv: torch.Tensor, - score: torch.Tensor, - ape: torch.Tensor, - positions: torch.Tensor, - state_cache: torch.Tensor, - slot_mapping: torch.Tensor, - block_size: int, - state_width: int, - compress_ratio: int, - pdl_kwargs: dict | None = None, -) -> None: - """Write packed [kv, score+ape] partial states into the compressor cache. - - One program per token; pads (slot_id == -1) are skipped. - """ - num_actual = slot_mapping.shape[0] - head_size = kv.shape[-1] - _save_partial_states_kernel[(num_actual,)]( - kv, - kv.stride(0), - score, - score.stride(0), - ape, - ape.stride(0), - positions, - state_cache, - state_cache.stride(0), - state_cache.stride(1), - slot_mapping, - block_size, - HEAD_SIZE=head_size, - TRITON_BLOCK_SIZE=triton.next_power_of_2(head_size), - STATE_WIDTH=state_width, - COMPRESS_RATIO=compress_ratio, - **(pdl_kwargs or {}), - ) - - -@triton.jit -def _save_partial_states_kernel( - kv_ptr, - kv_stride, - score_ptr, - score_stride, - ape_ptr, - ape_stride, - positions_ptr, - state_cache_ptr, - state_cache_stride0, - state_cache_stride1, - slot_mapping_ptr, - block_size, - HEAD_SIZE: tl.constexpr, - TRITON_BLOCK_SIZE: tl.constexpr, - # state_cache last dim packs [kv_state, score_state], each STATE_WIDTH wide. - STATE_WIDTH: tl.constexpr, - COMPRESS_RATIO: tl.constexpr, +class SavePartialStatesKernel( + VllmTritonJitKernel["SavePartialStatesKernel.CompileKey"] ): - token_idx = tl.program_id(0) - slot_id = tl.load(slot_mapping_ptr + token_idx) + @dataclass(frozen=True) + class CompileKey: + head_size: int + triton_block_size: int + state_width: int + compress_ratio: int + kv_stride: int + score_stride: int + ape_stride: int + state_cache_stride0: int + state_cache_stride1: int + block_size: int - # Skip padded / invalid tokens (slot_id == -1 is the PAD sentinel used - # by vLLM). During CUDA graph replay the batch may contain padding - # tokens whose slot_mapping is -1; writing to kv_state[-1] would be an - # illegal memory access. - if slot_id < 0: - return + @staticmethod + @triton.jit + def kernel( + kv_ptr, + kv_stride, + score_ptr, + score_stride, + ape_ptr, + ape_stride, + positions_ptr, + state_cache_ptr, + state_cache_stride0, + state_cache_stride1, + slot_mapping_ptr, + block_size, + HEAD_SIZE: tl.constexpr, + TRITON_BLOCK_SIZE: tl.constexpr, + # state_cache last dim packs [kv_state, score_state], each STATE_WIDTH wide. + STATE_WIDTH: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, + ): + token_idx = tl.program_id(0) + slot_id = tl.load(slot_mapping_ptr + token_idx) - block_idx = slot_id // block_size - pos_in_block = slot_id % block_size - base_ptr = ( - state_cache_ptr - + block_idx * state_cache_stride0 - + pos_in_block * state_cache_stride1 - ) + # Skip padded / invalid tokens (slot_id == -1 is the PAD sentinel used + # by vLLM). During CUDA graph replay the batch may contain padding + # tokens whose slot_mapping is -1; writing to kv_state[-1] would be an + # illegal memory access. + if slot_id < 0: + return - block = tl.arange(0, TRITON_BLOCK_SIZE) - mask = block < HEAD_SIZE + block_idx = slot_id // block_size + pos_in_block = slot_id % block_size + base_ptr = ( + state_cache_ptr + + block_idx * state_cache_stride0 + + pos_in_block * state_cache_stride1 + ) - kv = tl.load(kv_ptr + token_idx * kv_stride + block, mask=mask) - tl.store(base_ptr + block, kv, mask=mask) + block = tl.arange(0, TRITON_BLOCK_SIZE) + mask = block < HEAD_SIZE - # Fused: score += ape[position % compress_ratio] - position = tl.load(positions_ptr + token_idx) - ape_row = position % COMPRESS_RATIO - ape = tl.load(ape_ptr + ape_row * ape_stride + block, mask=mask) - score = tl.load(score_ptr + token_idx * score_stride + block, mask=mask) - tl.store( - base_ptr + STATE_WIDTH + block, - score + ape, - mask=mask, - ) + kv = tl.load(kv_ptr + token_idx * kv_stride + block, mask=mask) + tl.store(base_ptr + block, kv, mask=mask) + + # Fused: score += ape[position % compress_ratio] + position = tl.load(positions_ptr + token_idx) + ape_row = position % COMPRESS_RATIO + ape = tl.load(ape_ptr + ape_row * ape_stride + block, mask=mask) + score = tl.load(score_ptr + token_idx * score_stride + block, mask=mask) + tl.store( + base_ptr + STATE_WIDTH + block, + score + ape, + mask=mask, + ) + + def dispatch( # type: ignore[override] + self, + *, + head_size: int, + **compile_key_fields: int, + ) -> CompileKey: + return self.CompileKey( + **compile_key_fields, + head_size=head_size, + triton_block_size=next_power_of_2(head_size), + ) + + def get_warmup_keys( + self, + *, + head_dim: int, + compress_ratio: int, + ) -> list[CompileKey]: + if head_dim <= 0 or compress_ratio not in (4, 128): + return [] + + coefficient = 2 if compress_ratio == 4 else 1 + return self._trace_dispatch(self.dispatch)( + head_size=coefficient * head_dim, + state_width=coefficient * head_dim, + compress_ratio=compress_ratio, + kv_stride=2 * coefficient * head_dim, + score_stride=2 * coefficient * head_dim, + ape_stride=coefficient * head_dim, + state_cache_stride0=8 * coefficient * head_dim, + state_cache_stride1=2 * coefficient * head_dim, + block_size=4 if compress_ratio == 4 else 8, + ) + + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + return dict( + kv=TritonWarmupTensor( + torch.float32, + shape=(1, compile_key.head_size), + strides=(compile_key.kv_stride, 1), + ), + score=TritonWarmupTensor( + torch.float32, + shape=(1, 1), + strides=(compile_key.score_stride, 1), + ), + ape=TritonWarmupTensor( + torch.float32, + shape=(1, 1), + strides=(compile_key.ape_stride, 1), + ), + positions=TritonWarmupTensor(torch.int64), + state_cache=TritonWarmupTensor( + torch.float32, + shape=(1, 1, compile_key.state_width), + strides=( + compile_key.state_cache_stride0, + compile_key.state_cache_stride1, + 1, + ), + ), + slot_mapping=TritonWarmupTensor(torch.int64), + block_size=compile_key.block_size, + state_width=compile_key.state_width, + compress_ratio=compile_key.compress_ratio, + ) + + @kernel_launcher + def __call__( + self, + kv: torch.Tensor, + score: torch.Tensor, + ape: torch.Tensor, + positions: torch.Tensor, + state_cache: torch.Tensor, + slot_mapping: torch.Tensor, + block_size: int, + state_width: int, + compress_ratio: int, + pdl_kwargs: dict | None = None, + ) -> LaunchSpec: + """Write packed [kv, score+ape] partial states into the compressor cache. + + One program per token; pads (slot_id == -1) are skipped. + """ + num_actual = slot_mapping.shape[0] + head_size = kv.shape[-1] + return (num_actual,), dict( + kv_stride=kv.stride(0), + score_stride=score.stride(0), + ape_stride=ape.stride(0), + state_cache_stride0=state_cache.stride(0), + state_cache_stride1=state_cache.stride(1), + block_size=block_size, + HEAD_SIZE=head_size, + TRITON_BLOCK_SIZE=next_power_of_2(head_size), + STATE_WIDTH=state_width, + COMPRESS_RATIO=compress_ratio, + **(pdl_kwargs or {}), + ) + + +_SAVE_PARTIAL_STATES_KERNEL = SavePartialStatesKernel() diff --git a/vllm/models/deepseek_v4/compressor.py b/vllm/models/deepseek_v4/compressor.py index ccfdbeaa43..3bc636945a 100644 --- a/vllm/models/deepseek_v4/compressor.py +++ b/vllm/models/deepseek_v4/compressor.py @@ -18,7 +18,7 @@ from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import ( ) from vllm.models.deepseek_v4.common.ops.fused_indexer_q import MXFP4_BLOCK_SIZE from vllm.models.deepseek_v4.common.ops.save_partial_states import ( - save_partial_states, + _SAVE_PARTIAL_STATES_KERNEL, ) from vllm.platforms import current_platform from vllm.v1.attention.backend import ( @@ -306,6 +306,18 @@ class DeepseekCompressor(nn.Module): f"Unsupported head_dim for fused quant+cache: {self.head_dim}" ) + if vllm_config.kernel_config.enable_jit_warmup: + _SAVE_PARTIAL_STATES_KERNEL.register_warmup( + head_dim=self.head_dim, + compress_ratio=self.compress_ratio, + ) + if self.head_dim != 512: + from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import ( # noqa: E501 + _FUSED_KV_COMPRESS_NORM_ROPE_INSERT_INDEXER_TRITON_KERNEL, + ) + + _FUSED_KV_COMPRESS_NORM_ROPE_INSERT_INDEXER_TRITON_KERNEL.register_warmup() + def forward( self, # [num_tokens, 2 * self.coff * self.head_dim] @@ -351,7 +363,7 @@ class DeepseekCompressor(nn.Module): # GEMM; state_cache from this kernel) but neither emits/waits on PDL # grid dependency primitives, so launch_pdl=True caused a # read-after-write race and non-deterministic output. - save_partial_states( + _SAVE_PARTIAL_STATES_KERNEL( kv=kv, score=score, ape=self.ape, diff --git a/vllm/models/deepseek_v4/nvidia/mtp.py b/vllm/models/deepseek_v4/nvidia/mtp.py index b35ec6481a..55403559dc 100644 --- a/vllm/models/deepseek_v4/nvidia/mtp.py +++ b/vllm/models/deepseek_v4/nvidia/mtp.py @@ -51,9 +51,9 @@ from vllm.models.common.ops.sequence_parallel import ( sp_padding_mask, sp_shard, ) -from vllm.models.deepseek_v4.common.ops import ( - fused_mtp_input_rmsnorm, - mtp_shared_head_rmsnorm, +from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import ( + _FUSED_MTP_INPUT_RMSNORM_KERNEL, + _MTP_SHARED_HEAD_RMSNORM_KERNEL, ) from vllm.sequence import IntermediateTensors @@ -141,6 +141,10 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module): aux_stream_list=aux_stream_list, ) + if vllm_config.kernel_config.enable_jit_warmup: + _FUSED_MTP_INPUT_RMSNORM_KERNEL.register_warmup() + _MTP_SHARED_HEAD_RMSNORM_KERNEL.register_warmup() + def forward( self, input_ids: torch.Tensor, @@ -157,7 +161,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module): -1, self.hc_mult, self.config.hidden_size ) # Fused: mask inputs at position 0 (not needed by MTP), enorm, hnorm. - inputs_embeds, previous_hidden_states = fused_mtp_input_rmsnorm( + inputs_embeds, previous_hidden_states = _FUSED_MTP_INPUT_RMSNORM_KERNEL( inputs_embeds, positions, previous_hidden_states, @@ -270,7 +274,7 @@ class DeepSeekV4MultiTokenPredictor(nn.Module): mtp_layer.rms_norm_eps, mtp_layer.hc_eps, ) - hidden_states = mtp_shared_head_rmsnorm( + hidden_states = _MTP_SHARED_HEAD_RMSNORM_KERNEL( hidden_states, mtp_layer.shared_head.norm.weight.data, mtp_layer.shared_head.norm.variance_epsilon, diff --git a/vllm/models/deepseek_v4/sparse_mla.py b/vllm/models/deepseek_v4/sparse_mla.py index 7f371c3922..81659f342a 100644 --- a/vllm/models/deepseek_v4/sparse_mla.py +++ b/vllm/models/deepseek_v4/sparse_mla.py @@ -9,6 +9,12 @@ import torch from vllm.config import VllmConfig from vllm.config.cache import CacheDType +from vllm.model_executor.warmup.jit_warmup_triton_helper import ( + LaunchSpec, + TritonWarmupTensor, + VllmTritonJitKernel, + kernel_launcher, +) from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability from vllm.triton_utils import tl, triton @@ -338,90 +344,180 @@ def build_c128a_topk_metadata( if num_tokens == 0: return global_decode, decode_lens, prefill_local - _build_c128a_topk_metadata_kernel[(num_tokens,)]( + _BUILD_C128A_TOPK_METADATA_KERNEL( global_decode_buffer, - global_decode_buffer.stride(0), decode_lens_buffer, prefill_buffer, - prefill_buffer.stride(0), positions, compress_ratio, max_compressed_tokens, num_decode_tokens, token_to_req_indices, block_table, - block_table.stride(0), block_size, slot_mapping, - BLOCK_SIZE=1024, ) return global_decode, decode_lens, prefill_local -@triton.jit -def _build_c128a_topk_metadata_kernel( - # Decode outputs - global_decode_ptr, - global_decode_stride, - decode_lens_ptr, - # Prefill output - prefill_local_ptr, - prefill_local_stride, - # Inputs - positions_ptr, - compress_ratio, - max_compressed_tokens, - num_decode_tokens, - token_to_req_indices_ptr, - block_table_ptr, - block_table_stride, - block_size, - slot_mapping_ptr, - BLOCK_SIZE: tl.constexpr, +class BuildC128ATopkMetadataKernel( + VllmTritonJitKernel["BuildC128ATopkMetadataKernel.CompileKey"] ): - token_idx = tl.program_id(0) - position = tl.load(positions_ptr + token_idx) - num_compressed = (position + 1) // compress_ratio - num_compressed = tl.minimum(num_compressed, max_compressed_tokens) - is_decode = token_idx < num_decode_tokens + @dataclass(frozen=True) + class CompileKey: + compress_ratio: int + max_compressed_tokens: int + block_size: int + triton_block_size: int - if is_decode: - # --- Decode: block-table lookup → global slot ids + count --- - is_valid_token = tl.load(slot_mapping_ptr + token_idx) >= 0 - req_idx = tl.load(token_to_req_indices_ptr + token_idx) - count = tl.zeros((), dtype=tl.int32) - for i in range(0, max_compressed_tokens, BLOCK_SIZE): - offset = i + tl.arange(0, BLOCK_SIZE) - mask = offset < max_compressed_tokens - is_valid = offset < num_compressed + @staticmethod + @triton.jit( + do_not_specialize=[ + "global_decode_stride", + "prefill_local_stride", + "num_decode_tokens", + "block_table_stride", + ] + ) + def kernel( + # Decode outputs + global_decode_ptr, + global_decode_stride, + decode_lens_ptr, + # Prefill output + prefill_local_ptr, + prefill_local_stride, + # Inputs + positions_ptr, + compress_ratio, + max_compressed_tokens, + num_decode_tokens, + token_to_req_indices_ptr, + block_table_ptr, + block_table_stride, + block_size, + slot_mapping_ptr, + BLOCK_SIZE: tl.constexpr, + ): + token_idx = tl.program_id(0) + position = tl.load(positions_ptr + token_idx) + num_compressed = (position + 1) // compress_ratio + num_compressed = tl.minimum(num_compressed, max_compressed_tokens) + is_decode = token_idx < num_decode_tokens + + if is_decode: + # --- Decode: block-table lookup → global slot ids + count --- + is_valid_token = tl.load(slot_mapping_ptr + token_idx) >= 0 + req_idx = tl.load(token_to_req_indices_ptr + token_idx) + count = tl.zeros((), dtype=tl.int32) + for i in range(0, max_compressed_tokens, BLOCK_SIZE): + offset = i + tl.arange(0, BLOCK_SIZE) + mask = offset < max_compressed_tokens + is_valid = offset < num_compressed + + block_indices = offset // block_size + block_numbers = tl.load( + block_table_ptr + req_idx * block_table_stride + block_indices, + mask=mask & is_valid, + ) + block_offsets = offset % block_size + slot_ids = block_numbers * block_size + block_offsets + slot_ids = tl.where(is_valid, slot_ids, -1) + tl.store( + global_decode_ptr + token_idx * global_decode_stride + offset, + slot_ids, + mask=mask, + ) + count += tl.sum(is_valid.to(tl.int32), axis=0) - block_indices = offset // block_size - block_numbers = tl.load( - block_table_ptr + req_idx * block_table_stride + block_indices, - mask=mask & is_valid, - ) - block_offsets = offset % block_size - slot_ids = block_numbers * block_size + block_offsets - slot_ids = tl.where(is_valid, slot_ids, -1) tl.store( - global_decode_ptr + token_idx * global_decode_stride + offset, - slot_ids, - mask=mask, + decode_lens_ptr + token_idx, + tl.where(is_valid_token, count, 0), ) - count += tl.sum(is_valid.to(tl.int32), axis=0) + else: + # --- Prefill: write local indices --- + pfx_idx = token_idx - num_decode_tokens + for i in range(0, max_compressed_tokens, BLOCK_SIZE): + offset = i + tl.arange(0, BLOCK_SIZE) + mask = offset < max_compressed_tokens + tl.store( + prefill_local_ptr + pfx_idx * prefill_local_stride + offset, + tl.where(offset < num_compressed, offset, -1), + mask=mask, + ) - tl.store( - decode_lens_ptr + token_idx, - tl.where(is_valid_token, count, 0), + def dispatch( # type: ignore[override] + self, + **compile_key_fields: int, + ) -> CompileKey: + return self.CompileKey(**compile_key_fields) + + def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]: + hf_config = vllm_config.model_config.hf_config + if getattr(hf_config, "model_type", None) != "deepseek_v4": + return [] + + compress_ratio = 128 + max_model_len = vllm_config.model_config.max_model_len + if max_model_len <= 0: + return [] + max_compressed_tokens = cdiv(max_model_len, compress_ratio) + max_compressed_tokens = ( + cdiv(max_compressed_tokens, _C128A_TOPK_ALIGNMENT) * _C128A_TOPK_ALIGNMENT ) - else: - # --- Prefill: write local indices --- - pfx_idx = token_idx - num_decode_tokens - for i in range(0, max_compressed_tokens, BLOCK_SIZE): - offset = i + tl.arange(0, BLOCK_SIZE) - mask = offset < max_compressed_tokens - tl.store( - prefill_local_ptr + pfx_idx * prefill_local_stride + offset, - tl.where(offset < num_compressed, offset, -1), - mask=mask, - ) + return self._trace_dispatch(self.dispatch)( + compress_ratio=compress_ratio, + max_compressed_tokens=max_compressed_tokens, + # DeepSeek V4 sparse MLA uses 256-token KV pages; C128A metadata + # works in compressed-token units. + block_size=256 // compress_ratio, + triton_block_size=1024, + ) + + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + int32_ptr = TritonWarmupTensor(torch.int32) + return dict( + global_decode_buffer=TritonWarmupTensor( + torch.int32, shape=(1, 1), strides=(1, 1) + ), + decode_lens_buffer=int32_ptr, + prefill_buffer=TritonWarmupTensor( + torch.int32, shape=(1, 1), strides=(1, 1) + ), + positions=TritonWarmupTensor(torch.int64), + compress_ratio=compile_key.compress_ratio, + max_compressed_tokens=compile_key.max_compressed_tokens, + num_decode_tokens=0, + token_to_req_indices=int32_ptr, + block_table=TritonWarmupTensor(torch.int32, shape=(1, 1), strides=(1, 1)), + block_size=compile_key.block_size, + slot_mapping=TritonWarmupTensor(torch.int64), + ) + + @kernel_launcher + def __call__( + self, + global_decode_buffer: torch.Tensor, + decode_lens_buffer: torch.Tensor, + prefill_buffer: torch.Tensor, + positions: torch.Tensor, + compress_ratio: int, + max_compressed_tokens: int, + num_decode_tokens: int, + token_to_req_indices: torch.Tensor, + block_table: torch.Tensor, + block_size: int, + slot_mapping: torch.Tensor, + ) -> LaunchSpec: + return (positions.shape[0],), dict( + global_decode_ptr=global_decode_buffer, + global_decode_stride=global_decode_buffer.stride(0), + decode_lens_ptr=decode_lens_buffer, + prefill_local_ptr=prefill_buffer, + prefill_local_stride=prefill_buffer.stride(0), + block_table_stride=block_table.stride(0), + BLOCK_SIZE=1024, + ) + + +_BUILD_C128A_TOPK_METADATA_KERNEL = BuildC128ATopkMetadataKernel() diff --git a/vllm/models/deepseek_v4/xpu/mtp.py b/vllm/models/deepseek_v4/xpu/mtp.py index 5c905ac22e..3d1ce97687 100644 --- a/vllm/models/deepseek_v4/xpu/mtp.py +++ b/vllm/models/deepseek_v4/xpu/mtp.py @@ -39,9 +39,9 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.deepseek_mtp import SharedHead from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weight_name from vllm.model_executor.models.utils import maybe_prefix -from vllm.models.deepseek_v4.common.ops import ( - fused_mtp_input_rmsnorm, - mtp_shared_head_rmsnorm, +from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import ( + _FUSED_MTP_INPUT_RMSNORM_KERNEL, + _MTP_SHARED_HEAD_RMSNORM_KERNEL, ) from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors @@ -130,6 +130,10 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module): self.hc_head_op = HCHeadOp() + if vllm_config.kernel_config.enable_jit_warmup: + _FUSED_MTP_INPUT_RMSNORM_KERNEL.register_warmup() + _MTP_SHARED_HEAD_RMSNORM_KERNEL.register_warmup() + def forward( self, input_ids: torch.Tensor, @@ -146,7 +150,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module): -1, self.hc_mult, self.config.hidden_size ) # Fused: mask inputs at position 0 (not needed by MTP), enorm, hnorm. - inputs_embeds, previous_hidden_states = fused_mtp_input_rmsnorm( + inputs_embeds, previous_hidden_states = _FUSED_MTP_INPUT_RMSNORM_KERNEL( inputs_embeds, positions, previous_hidden_states, @@ -253,7 +257,7 @@ class DeepSeekV4MultiTokenPredictor(nn.Module): mtp_layer.rms_norm_eps, mtp_layer.hc_eps, ) - hidden_states = mtp_shared_head_rmsnorm( + hidden_states = _MTP_SHARED_HEAD_RMSNORM_KERNEL( hidden_states, mtp_layer.shared_head.norm.weight.data, mtp_layer.shared_head.norm.variance_epsilon,