[4/N][warmup][DSv4] Migrate common attention kernels (#50176)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Codex
parent
f998862d46
commit
6ddbab03de
@@ -13,7 +13,9 @@ from __future__ import annotations
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.models.common.ops.fused_qk_rmsnorm import (
|
||||
fused_q_kv_rmsnorm,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
|
||||
@@ -431,6 +431,40 @@ def test_dispatch_can_forward_compile_key_fields() -> None:
|
||||
kernel.compile_key({"tokens": 5, "mode": 2, "extra": 1})
|
||||
|
||||
|
||||
def test_forwarded_compile_key_fields_exclude_predicate_only_inputs() -> None:
|
||||
class ForwardingKernel(VllmJitKernel["ForwardingKernel.CompileKey"]):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
mode: int
|
||||
block_size: int
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
tokens: int,
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
**compile_key_fields,
|
||||
block_size=_round_up(tokens, multiple=4),
|
||||
)
|
||||
|
||||
def get_warmup_keys(self) -> list[CompileKey]:
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
tokens=(1, 5),
|
||||
mode=2,
|
||||
max_tokens=4,
|
||||
_when=lambda *, tokens, max_tokens: tokens <= max_tokens,
|
||||
)
|
||||
|
||||
def compile(self, compile_key: CompileKey) -> None:
|
||||
pass
|
||||
|
||||
assert ForwardingKernel().get_warmup_keys() == [
|
||||
ForwardingKernel.CompileKey(mode=2, block_size=4)
|
||||
]
|
||||
|
||||
|
||||
def test_dispatch_supports_tuple_and_mapping_subscriptions() -> None:
|
||||
class SubscriptKernel(VllmJitKernel["SubscriptKernel.CompileKey"]):
|
||||
@dataclass(frozen=True)
|
||||
|
||||
@@ -14,8 +14,8 @@ from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Generic, TypeVar
|
||||
from dataclasses import dataclass, fields
|
||||
from typing import Any, Generic, TypeVar, cast
|
||||
|
||||
__all__ = [
|
||||
"JitWarmupRegistry",
|
||||
@@ -597,6 +597,16 @@ class VllmJitKernel(Generic[CompileKeyT], ABC):
|
||||
predicate_trace = (
|
||||
_trace_warmup_predicate(_when) if _when is not None else None
|
||||
)
|
||||
predicate_only_names: frozenset[str] = frozenset()
|
||||
if predicate_trace is not None:
|
||||
compile_key_fields = frozenset(
|
||||
field.name for field in fields(cast(Any, self.CompileKey))
|
||||
)
|
||||
predicate_only_names = (
|
||||
predicate_trace.input_names
|
||||
- compile_key_dispatch_trace.input_names
|
||||
- compile_key_fields
|
||||
)
|
||||
# Unmatched **kwargs fields also belong to the expansion space.
|
||||
available_names = set(kwargs).union(
|
||||
*(group.rows[0] for group in input_groups)
|
||||
@@ -638,7 +648,12 @@ class VllmJitKernel(Generic[CompileKeyT], ABC):
|
||||
):
|
||||
continue
|
||||
compile_key = compile_key_dispatch_trace.compile_key(
|
||||
self.CompileKey, dispatch_values
|
||||
self.CompileKey,
|
||||
{
|
||||
name: value
|
||||
for name, value in dispatch_values.items()
|
||||
if name not in predicate_only_names
|
||||
},
|
||||
)
|
||||
compile_keys[compile_key] = None
|
||||
return list(compile_keys)
|
||||
|
||||
@@ -15,7 +15,13 @@ from vllm.model_executor.warmup.jit_warmup import (
|
||||
)
|
||||
|
||||
CompileKeyT = TypeVar("CompileKeyT")
|
||||
LaunchSpec = tuple[tuple[int, ...], dict[str, Any]]
|
||||
# ``(grid, launch_kwargs)`` or ``(grid, launch_kwargs, outputs)`` for
|
||||
# self-allocating kernels. ``grid=None`` skips the launch (e.g. empty batch)
|
||||
# but still returns the declared outputs.
|
||||
LaunchSpec = (
|
||||
tuple[tuple[int, ...] | None, dict[str, Any]]
|
||||
| tuple[tuple[int, ...] | None, dict[str, Any], Any]
|
||||
)
|
||||
|
||||
|
||||
def triton_scalar_specialization_rep(value: int) -> int:
|
||||
@@ -91,6 +97,7 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
|
||||
kernel: ClassVar[Any]
|
||||
_warming = False
|
||||
_warming_compile_key: CompileKeyT | None = None
|
||||
|
||||
@abstractmethod
|
||||
def warmup_inputs(self, compile_key: CompileKeyT) -> dict[str, Any]:
|
||||
@@ -100,10 +107,12 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
def compile(self, compile_key: CompileKeyT) -> None:
|
||||
inputs = self.warmup_inputs(compile_key)
|
||||
self._warming = True
|
||||
self._warming_compile_key = compile_key
|
||||
try:
|
||||
cast(Callable[..., None], self)(**inputs)
|
||||
finally:
|
||||
self._warming = False
|
||||
self._warming_compile_key = None
|
||||
|
||||
@cached_property
|
||||
def _kernel_arg_names(self) -> tuple[str, ...]:
|
||||
@@ -119,11 +128,14 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
|
||||
def launch(
|
||||
self,
|
||||
grid: tuple[int, ...],
|
||||
grid: tuple[int, ...] | None,
|
||||
inputs: Mapping[str, Any],
|
||||
/,
|
||||
*,
|
||||
kernel: Any = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
kernel = kernel if kernel is not None else self.kernel
|
||||
runtime_launcher = kwargs.pop("_runtime_launcher", None)
|
||||
runtime_launcher_arg_count = kwargs.pop("_runtime_launcher_arg_count", 0)
|
||||
for name, value in inputs.items():
|
||||
@@ -131,22 +143,36 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
if target in self._kernel_arg_names and target not in kwargs:
|
||||
kwargs[target] = value
|
||||
if self._warming:
|
||||
warmup = getattr(self.kernel, "warmup", None)
|
||||
if (
|
||||
self._warming_compile_key is not None
|
||||
and "launch_pdl" in kwargs
|
||||
and hasattr(self._warming_compile_key, "launch_pdl")
|
||||
):
|
||||
kwargs["launch_pdl"] = self._warming_compile_key.launch_pdl
|
||||
warmup = getattr(kernel, "warmup", None)
|
||||
assert warmup is not None
|
||||
return warmup(grid=(1,), **kwargs)
|
||||
if grid is None:
|
||||
return None
|
||||
if runtime_launcher is not None:
|
||||
regular_args = [
|
||||
kwargs.pop(name)
|
||||
for name in self._kernel_arg_names[:runtime_launcher_arg_count]
|
||||
]
|
||||
return runtime_launcher(self.kernel, grid, *regular_args, **kwargs)
|
||||
return self.kernel[grid](**kwargs)
|
||||
return runtime_launcher(kernel, grid, *regular_args, **kwargs)
|
||||
return kernel[grid](**kwargs)
|
||||
|
||||
|
||||
def kernel_launcher(
|
||||
call_fn: Callable[..., LaunchSpec],
|
||||
) -> Callable[..., None]:
|
||||
"""Launch a Triton kernel from a declarative ``__call__`` specification."""
|
||||
) -> Callable[..., Any]:
|
||||
"""Launch a Triton kernel from a declarative ``__call__`` specification.
|
||||
|
||||
``call_fn`` returns either ``(grid, launch_kwargs)`` or, when it allocates
|
||||
its own outputs, ``(grid, launch_kwargs, outputs)``. The declared outputs are
|
||||
returned to the caller. A ``grid`` of ``None`` skips the launch (e.g. an
|
||||
empty-token batch) while still returning the outputs.
|
||||
"""
|
||||
signature = inspect.signature(call_fn)
|
||||
|
||||
@wraps(call_fn)
|
||||
@@ -154,14 +180,20 @@ def kernel_launcher(
|
||||
self: VllmTritonJitKernel[Any],
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
grid, launch_kwargs = call_fn(self, *args, **kwargs)
|
||||
) -> Any:
|
||||
spec = call_fn(self, *args, **kwargs)
|
||||
if len(spec) == 3:
|
||||
grid, launch_kwargs, outputs = spec
|
||||
else:
|
||||
grid, launch_kwargs = spec
|
||||
outputs = None
|
||||
bound = signature.bind(self, *args, **kwargs)
|
||||
bound.apply_defaults()
|
||||
inputs = {
|
||||
name: value for name, value in bound.arguments.items() if name != "self"
|
||||
}
|
||||
self.launch(grid, inputs, **launch_kwargs)
|
||||
return outputs
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Ops shared across model implementations."""
|
||||
|
||||
from .fused_qk_rmsnorm import fused_q_kv_rmsnorm
|
||||
from .fused_qk_rmsnorm import _FUSED_Q_KV_RMSNORM_KERNEL, fused_q_kv_rmsnorm
|
||||
|
||||
__all__ = [
|
||||
"_FUSED_Q_KV_RMSNORM_KERNEL",
|
||||
"fused_q_kv_rmsnorm",
|
||||
]
|
||||
|
||||
@@ -1,67 +1,194 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
kernel_launcher,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_q_kv_rmsnorm_kernel(
|
||||
q_ptr,
|
||||
q_out_ptr,
|
||||
q_weight_ptr,
|
||||
q_in_stride,
|
||||
q_out_stride,
|
||||
kv_ptr,
|
||||
kv_out_ptr,
|
||||
kv_weight_ptr,
|
||||
kv_in_stride,
|
||||
kv_out_stride,
|
||||
eps,
|
||||
Q_SIZE: tl.constexpr,
|
||||
KV_SIZE: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
# num_tokens goes on grid-x (max 2**31 - 1); task goes on grid-y.
|
||||
# CUDA's grid-y/z are capped at 65535, so putting num_tokens there crashes
|
||||
# the launch at max-num-batched-tokens >= 65536 with "invalid argument".
|
||||
# int64: q_in_stride can be ~24K (128 heads × 192) and overflows int32
|
||||
# past num_tokens ~87K under large chunked prefill.
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
pid_task = tl.program_id(1)
|
||||
class FusedQKVRMSNormKernel(VllmTritonJitKernel["FusedQKVRMSNormKernel.CompileKey"]):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
dtype: torch.dtype
|
||||
q_size: int
|
||||
kv_size: int
|
||||
block_size: int
|
||||
q_in_stride: int
|
||||
q_out_stride: int
|
||||
kv_in_stride: int
|
||||
kv_out_stride: int
|
||||
eps: float
|
||||
launch_pdl: bool
|
||||
|
||||
if pid_task == 0:
|
||||
SIZE = Q_SIZE
|
||||
row_in = q_ptr + token_idx * q_in_stride
|
||||
weight_ptr = q_weight_ptr
|
||||
row_out = q_out_ptr + token_idx * q_out_stride
|
||||
else:
|
||||
SIZE = KV_SIZE
|
||||
row_in = kv_ptr + token_idx * kv_in_stride
|
||||
weight_ptr = kv_weight_ptr
|
||||
row_out = kv_out_ptr + token_idx * kv_out_stride
|
||||
@staticmethod
|
||||
@triton.jit
|
||||
def kernel(
|
||||
q_ptr,
|
||||
q_out_ptr,
|
||||
q_weight_ptr,
|
||||
q_in_stride,
|
||||
q_out_stride,
|
||||
kv_ptr,
|
||||
kv_out_ptr,
|
||||
kv_weight_ptr,
|
||||
kv_in_stride,
|
||||
kv_out_stride,
|
||||
eps,
|
||||
Q_SIZE: tl.constexpr,
|
||||
KV_SIZE: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
# num_tokens goes on grid-x (max 2**31 - 1); task goes on grid-y.
|
||||
# CUDA's grid-y/z are capped at 65535, so putting num_tokens there crashes
|
||||
# the launch at max-num-batched-tokens >= 65536 with "invalid argument".
|
||||
# int64: q_in_stride can be ~24K (128 heads × 192) and overflows int32
|
||||
# past num_tokens ~87K under large chunked prefill.
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
pid_task = tl.program_id(1)
|
||||
|
||||
# RMSNorm in fp32 throughout — matches csrc/layernorm_kernels.cu's
|
||||
# `(scalar_t)(x * s_variance * w)` and DeepseekV4's compressor kernel, which
|
||||
# keep x, rrms, and w all in fp32 and perform a single cast at store.
|
||||
block = tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < SIZE
|
||||
# The weight load does not depend on the producer's output, so issue it
|
||||
# before the PDL wait: gamma streams in while the producer finishes,
|
||||
# leaving a single dependent global round trip (x) after the wait.
|
||||
w = tl.load(weight_ptr + block, mask=mask, other=0.0).to(tl.float32)
|
||||
if pid_task == 0:
|
||||
SIZE = Q_SIZE
|
||||
row_in = q_ptr + token_idx * q_in_stride
|
||||
weight_ptr = q_weight_ptr
|
||||
row_out = q_out_ptr + token_idx * q_out_stride
|
||||
else:
|
||||
SIZE = KV_SIZE
|
||||
row_in = kv_ptr + token_idx * kv_in_stride
|
||||
weight_ptr = kv_weight_ptr
|
||||
row_out = kv_out_ptr + token_idx * kv_out_stride
|
||||
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
# RMSNorm in fp32 throughout — matches csrc/layernorm_kernels.cu's
|
||||
# `(scalar_t)(x * s_variance * w)` and DeepseekV4's compressor kernel, which
|
||||
# keep x, rrms, and w all in fp32 and perform a single cast at store.
|
||||
block = tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < SIZE
|
||||
# The weight load does not depend on the producer's output, so issue it
|
||||
# before the PDL wait: gamma streams in while the producer finishes,
|
||||
# leaving a single dependent global round trip (x) after the wait.
|
||||
w = tl.load(weight_ptr + block, mask=mask, other=0.0).to(tl.float32)
|
||||
|
||||
x = tl.load(row_in + block, mask=mask, other=0.0).to(tl.float32)
|
||||
variance = tl.sum(x * x, axis=0) / SIZE
|
||||
rrms = tl.rsqrt(variance + eps)
|
||||
y = x * rrms * w
|
||||
tl.store(row_out + block, y.to(row_out.dtype.element_ty), mask=mask)
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
|
||||
x = tl.load(row_in + block, mask=mask, other=0.0).to(tl.float32)
|
||||
variance = tl.sum(x * x, axis=0) / SIZE
|
||||
rrms = tl.rsqrt(variance + eps)
|
||||
y = x * rrms * w
|
||||
tl.store(row_out + block, y.to(row_out.dtype.element_ty), mask=mask)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
q_size: int,
|
||||
kv_size: int,
|
||||
eps: float,
|
||||
launch_pdl: bool,
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
max_size = q_size if q_size >= kv_size else kv_size
|
||||
return self.CompileKey(
|
||||
**compile_key_fields,
|
||||
dtype=dtype,
|
||||
q_size=q_size,
|
||||
kv_size=kv_size,
|
||||
block_size=next_power_of_2(max_size),
|
||||
eps=eps,
|
||||
launch_pdl=launch_pdl,
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
q_size = int(getattr(hf_config, "q_lora_rank", 0) or 0)
|
||||
kv_size = int(getattr(hf_config, "head_dim", 0) or 0)
|
||||
if q_size <= 0 or kv_size <= 0:
|
||||
return []
|
||||
|
||||
input_stride = q_size + kv_size
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
q_size=q_size,
|
||||
kv_size=kv_size,
|
||||
q_in_stride=input_stride,
|
||||
q_out_stride=(input_stride, q_size),
|
||||
kv_in_stride=input_stride,
|
||||
kv_out_stride=(input_stride, kv_size),
|
||||
eps=float(hf_config.rms_norm_eps),
|
||||
launch_pdl=(False, True),
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
return dict(
|
||||
q=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.q_size),
|
||||
strides=(compile_key.q_in_stride, 1),
|
||||
),
|
||||
kv=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.kv_size),
|
||||
strides=(compile_key.kv_in_stride, 1),
|
||||
),
|
||||
q_out=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.q_size),
|
||||
strides=(compile_key.q_out_stride, 1),
|
||||
),
|
||||
kv_out=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.kv_size),
|
||||
strides=(compile_key.kv_out_stride, 1),
|
||||
),
|
||||
q_weight=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(compile_key.q_size,),
|
||||
),
|
||||
kv_weight=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(compile_key.kv_size,),
|
||||
),
|
||||
eps=compile_key.eps,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
q_out: torch.Tensor,
|
||||
kv_out: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
kv_weight: torch.Tensor,
|
||||
eps: float,
|
||||
) -> LaunchSpec:
|
||||
q_size = q.shape[1]
|
||||
kv_size = kv.shape[1]
|
||||
block_size = next_power_of_2(max(q_size, kv_size))
|
||||
return (q.shape[0], 2), dict(
|
||||
q_in_stride=q.stride(0),
|
||||
q_out_stride=q_out.stride(0),
|
||||
kv_in_stride=kv.stride(0),
|
||||
kv_out_stride=kv_out.stride(0),
|
||||
Q_SIZE=q_size,
|
||||
KV_SIZE=kv_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
# One vectorized 128-bit access per thread for K3/DSv4-sized rows,
|
||||
# instead of a serial per-thread chain at the default 4 warps.
|
||||
num_warps=8 if block_size >= 2048 else 4,
|
||||
)
|
||||
|
||||
|
||||
def fused_q_kv_rmsnorm(
|
||||
@@ -78,40 +205,19 @@ def fused_q_kv_rmsnorm(
|
||||
assert qr.stride(-1) == 1 and kv.stride(-1) == 1
|
||||
assert q_weight.is_contiguous() and kv_weight.is_contiguous()
|
||||
|
||||
q_size = qr.shape[1]
|
||||
kv_size = kv.shape[1]
|
||||
num_tokens = qr.shape[0]
|
||||
# Allocate packed outputs rather than empty_like: qr/kv are typically
|
||||
# column slices of a fused qkv_a projection, and for num_tokens == 1
|
||||
# empty_like preserves the parent's row stride (torch keeps strides of
|
||||
# size-1 dims), e.g. (2112, 1) for a [1, 1536] q slice. Downstream
|
||||
# dispatchers that require packed row-major inputs (e.g. the Kimi-K3
|
||||
# low-latency GEMM's _is_packed_row_major check) then reject the tensor
|
||||
# and silently fall back to cuBLAS on every decode step.
|
||||
qr_out = torch.empty(qr.shape, dtype=qr.dtype, device=qr.device)
|
||||
kv_out = torch.empty(kv.shape, dtype=kv.dtype, device=kv.device)
|
||||
if num_tokens == 0:
|
||||
return qr_out, kv_out
|
||||
|
||||
block_size = triton.next_power_of_2(max(q_size, kv_size))
|
||||
_fused_q_kv_rmsnorm_kernel[(num_tokens, 2)](
|
||||
qr,
|
||||
qr_out,
|
||||
q_weight,
|
||||
qr.stride(0),
|
||||
qr_out.stride(0),
|
||||
kv,
|
||||
kv_out,
|
||||
kv_weight,
|
||||
kv.stride(0),
|
||||
kv_out.stride(0),
|
||||
eps,
|
||||
Q_SIZE=q_size,
|
||||
KV_SIZE=kv_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
launch_pdl=current_platform.is_arch_support_pdl(),
|
||||
# One vectorized 128-bit access per thread for K3/DSv4-sized rows,
|
||||
# instead of a serial per-thread chain at the default 4 warps.
|
||||
num_warps=8 if block_size >= 2048 else 4,
|
||||
)
|
||||
if qr.shape[0] > 0:
|
||||
_FUSED_Q_KV_RMSNORM_KERNEL(
|
||||
qr,
|
||||
kv,
|
||||
qr_out,
|
||||
kv_out,
|
||||
q_weight,
|
||||
kv_weight,
|
||||
eps,
|
||||
)
|
||||
return qr_out, kv_out
|
||||
|
||||
|
||||
_FUSED_Q_KV_RMSNORM_KERNEL = FusedQKVRMSNormKernel()
|
||||
|
||||
@@ -39,9 +39,9 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.deepseek_mtp import SharedHead
|
||||
from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weight_name
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.models.deepseek_v4.common.ops import (
|
||||
fused_mtp_input_rmsnorm,
|
||||
mtp_shared_head_rmsnorm,
|
||||
from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import (
|
||||
_FUSED_MTP_INPUT_RMSNORM_KERNEL,
|
||||
_MTP_SHARED_HEAD_RMSNORM_KERNEL,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -127,6 +127,10 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
|
||||
self.hc_head_op = HCHeadOp()
|
||||
|
||||
if vllm_config.kernel_config.enable_jit_warmup:
|
||||
_FUSED_MTP_INPUT_RMSNORM_KERNEL.register_warmup()
|
||||
_MTP_SHARED_HEAD_RMSNORM_KERNEL.register_warmup()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
@@ -143,7 +147,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
-1, self.hc_mult, self.config.hidden_size
|
||||
)
|
||||
# Fused: mask inputs at position 0 (not needed by MTP), enorm, hnorm.
|
||||
inputs_embeds, previous_hidden_states = fused_mtp_input_rmsnorm(
|
||||
inputs_embeds, previous_hidden_states = _FUSED_MTP_INPUT_RMSNORM_KERNEL(
|
||||
inputs_embeds,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
@@ -256,7 +260,7 @@ class DeepSeekV4MultiTokenPredictor(nn.Module):
|
||||
mtp_layer.rms_norm_eps,
|
||||
mtp_layer.hc_eps,
|
||||
)
|
||||
hidden_states = mtp_shared_head_rmsnorm(
|
||||
hidden_states = _MTP_SHARED_HEAD_RMSNORM_KERNEL(
|
||||
hidden_states,
|
||||
mtp_layer.shared_head.norm.weight.data,
|
||||
mtp_layer.shared_head.norm.variance_epsilon,
|
||||
|
||||
@@ -22,7 +22,10 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.models.common.ops.fused_qk_rmsnorm import (
|
||||
_FUSED_Q_KV_RMSNORM_KERNEL,
|
||||
fused_q_kv_rmsnorm,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.ops import (
|
||||
fused_indexer_q_rope_quant,
|
||||
)
|
||||
@@ -417,6 +420,44 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC):
|
||||
|
||||
_COMBINE_TOPK_SWA_INDICES_KERNEL.register_warmup()
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
|
||||
_FUSED_Q_KV_RMSNORM_KERNEL.register_warmup()
|
||||
|
||||
backend_name = self.backend_cls.get_name()
|
||||
if current_platform.is_cuda():
|
||||
from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import ( # noqa: E501
|
||||
_FUSED_INV_ROPE_FP8_QUANT_KERNEL,
|
||||
)
|
||||
|
||||
_FUSED_INV_ROPE_FP8_QUANT_KERNEL.register_warmup()
|
||||
|
||||
if self.compress_ratio == 128:
|
||||
from vllm.models.deepseek_v4.sparse_mla import (
|
||||
_BUILD_C128A_TOPK_METADATA_KERNEL,
|
||||
)
|
||||
|
||||
_BUILD_C128A_TOPK_METADATA_KERNEL.register_warmup()
|
||||
|
||||
if backend_name == "FLASHMLA_SPARSE_DSV4":
|
||||
from vllm.models.deepseek_v4.common.ops.cache_utils import (
|
||||
_COMPUTE_GLOBAL_TOPK_INDICES_AND_LENS_KERNEL,
|
||||
_DEQUANTIZE_AND_GATHER_K_CACHE_KERNEL,
|
||||
)
|
||||
|
||||
_COMPUTE_GLOBAL_TOPK_INDICES_AND_LENS_KERNEL.register_warmup()
|
||||
if not has_cutedsl():
|
||||
_DEQUANTIZE_AND_GATHER_K_CACHE_KERNEL.register_warmup()
|
||||
elif backend_name == "FLASHINFER_MLA_SPARSE_DSV4":
|
||||
from vllm.models.deepseek_v4.common.ops.cache_utils import (
|
||||
_BUILD_FLASHINFER_MIXED_SPARSE_INDICES_KERNEL,
|
||||
_COMPUTE_GLOBAL_TOPK_INDICES_AND_LENS_KERNEL,
|
||||
)
|
||||
|
||||
_COMPUTE_GLOBAL_TOPK_INDICES_AND_LENS_KERNEL.register_warmup()
|
||||
_BUILD_FLASHINFER_MIXED_SPARSE_INDICES_KERNEL.register_warmup()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
@@ -974,6 +1015,22 @@ class DeepseekV4Indexer(nn.Module):
|
||||
torch.cuda.Event(),
|
||||
]
|
||||
|
||||
if vllm_config.kernel_config.enable_jit_warmup:
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
|
||||
if not has_cutedsl() and not current_platform.is_xpu():
|
||||
from vllm.models.deepseek_v4.common.ops.fused_indexer_q import (
|
||||
_FUSED_INDEXER_Q_ROPE_MXFP4_TRITON_KERNEL,
|
||||
_FUSED_INDEXER_Q_ROPE_QUANT_TRITON_KERNEL,
|
||||
)
|
||||
|
||||
(
|
||||
_FUSED_INDEXER_Q_ROPE_MXFP4_TRITON_KERNEL
|
||||
if self.use_fp4_kv
|
||||
else _FUSED_INDEXER_Q_ROPE_QUANT_TRITON_KERNEL
|
||||
).register_warmup()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
|
||||
@@ -10,8 +10,6 @@ from .cache_utils import (
|
||||
)
|
||||
from .fused_indexer_q import MXFP4_BLOCK_SIZE, fused_indexer_q_rope_quant
|
||||
from .fused_inv_rope_fp8_quant import fused_inv_rope_fp8_quant
|
||||
from .fused_mtp_input_rmsnorm import fused_mtp_input_rmsnorm, mtp_shared_head_rmsnorm
|
||||
from .save_partial_states import save_partial_states
|
||||
|
||||
__all__ = [
|
||||
"MXFP4_BLOCK_SIZE",
|
||||
@@ -21,8 +19,5 @@ __all__ = [
|
||||
"dequantize_and_gather_k_cache",
|
||||
"fused_indexer_q_rope_quant",
|
||||
"fused_inv_rope_fp8_quant",
|
||||
"fused_mtp_input_rmsnorm",
|
||||
"mtp_shared_head_rmsnorm",
|
||||
"quantize_and_insert_k_cache",
|
||||
"save_partial_states",
|
||||
]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -19,13 +19,22 @@ even/odd halves, producing (N_QUANT_BLOCKS, MXFP4_BLOCK/2) packed nibbles
|
||||
and N_QUANT_BLOCKS ue8m0 bytes.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
kernel_launcher,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import _ON_GFX950
|
||||
@@ -68,14 +77,32 @@ def compress_norm_rope_store_triton(
|
||||
kernel = _fused_kv_compress_norm_rope_insert_sparse_attn
|
||||
num_warps = 4
|
||||
kernel_kwargs = {"SANITIZE_CACHE_NANS": _ON_GFX950}
|
||||
elif use_fp4_cache:
|
||||
kernel = _fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn
|
||||
num_warps = 1
|
||||
kernel_kwargs = {}
|
||||
else:
|
||||
kernel = _fused_kv_compress_norm_rope_insert_indexer_attn
|
||||
num_warps = 1
|
||||
kernel_kwargs = {}
|
||||
_FUSED_KV_COMPRESS_NORM_ROPE_INSERT_INDEXER_TRITON_KERNEL(
|
||||
state_cache=state_cache,
|
||||
num_actual=num_actual,
|
||||
token_to_req_indices=token_to_req_indices,
|
||||
positions=positions,
|
||||
slot_mapping=slot_mapping,
|
||||
block_table=block_table,
|
||||
block_size=block_size,
|
||||
state_width=state_width,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
kv_cache=kv_cache,
|
||||
k_cache_metadata=k_cache_metadata,
|
||||
pdl_kwargs=pdl_kwargs,
|
||||
head_dim=head_dim,
|
||||
rope_head_dim=rope_head_dim,
|
||||
compress_ratio=compress_ratio,
|
||||
overlap=overlap,
|
||||
use_fp4_cache=use_fp4_cache,
|
||||
rms_norm_weight=rms_norm_weight,
|
||||
rms_norm_eps=rms_norm_eps,
|
||||
quant_block=quant_block,
|
||||
token_stride=token_stride,
|
||||
scale_dim=scale_dim,
|
||||
)
|
||||
return
|
||||
|
||||
kernel[(num_actual,)](
|
||||
# state cache
|
||||
@@ -673,7 +700,15 @@ def compress_norm_rope_store_two_stage_triton(
|
||||
# =============================================================================
|
||||
# Indexer path (head=128, all FP8, single quant block)
|
||||
# =============================================================================
|
||||
@triton.jit
|
||||
@triton.jit(
|
||||
do_not_specialize=[
|
||||
"state_cache_stride0",
|
||||
"state_cache_stride1",
|
||||
"block_table_stride",
|
||||
"rms_norm_eps",
|
||||
"cos_sin_stride",
|
||||
]
|
||||
)
|
||||
def _fused_kv_compress_norm_rope_insert_indexer_attn(
|
||||
# ── state cache (compressor internal state) ──
|
||||
state_cache_ptr,
|
||||
@@ -850,7 +885,15 @@ def _fused_kv_compress_norm_rope_insert_indexer_attn(
|
||||
# =============================================================================
|
||||
# Indexer path (head=128, MXFP4: 2 nibbles/byte + ue8m0 per 32-elem block)
|
||||
# =============================================================================
|
||||
@triton.jit
|
||||
@triton.jit(
|
||||
do_not_specialize=[
|
||||
"state_cache_stride0",
|
||||
"state_cache_stride1",
|
||||
"block_table_stride",
|
||||
"rms_norm_eps",
|
||||
"cos_sin_stride",
|
||||
]
|
||||
)
|
||||
def _fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn(
|
||||
# ── state cache (compressor internal state) ──
|
||||
state_cache_ptr,
|
||||
@@ -1038,3 +1081,232 @@ def _fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn(
|
||||
|
||||
tl.store(val_ptr + tl.arange(0, TOKEN_STRIDE), packed_flat)
|
||||
tl.store(scale_ptr + tl.arange(0, SCALE_DIM), ue8m0)
|
||||
|
||||
|
||||
class FusedKVCompressNormRopeInsertIndexerTritonKernel(
|
||||
VllmTritonJitKernel["FusedKVCompressNormRopeInsertIndexerTritonKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
dtype: torch.dtype
|
||||
use_fp4_cache: bool
|
||||
head_size: int
|
||||
triton_block_size: int
|
||||
state_width: int
|
||||
compress_ratio: int
|
||||
overlap: bool
|
||||
rope_head_dim: int
|
||||
fp8_max: float
|
||||
quant_block: int
|
||||
token_stride: int
|
||||
scale_dim: int
|
||||
block_size: int
|
||||
kv_cache_block_size: int
|
||||
kv_block_stride: int
|
||||
|
||||
# Canonical kernel for the launcher's arg_names mapping; the launched
|
||||
# variant (fp8/mxfp4, same arg_names) is picked per call in __call__.
|
||||
kernel = staticmethod(_fused_kv_compress_norm_rope_insert_indexer_attn)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
use_fp4_cache: bool,
|
||||
head_dim: int,
|
||||
rope_head_dim: int,
|
||||
compress_ratio: int,
|
||||
cache_block_size: int,
|
||||
cache_alignment: int,
|
||||
runtime_state_width: int | None = None,
|
||||
runtime_quant_block: int | None = None,
|
||||
runtime_token_stride: int | None = None,
|
||||
runtime_scale_dim: int | None = None,
|
||||
runtime_kv_block_stride: int | None = None,
|
||||
) -> CompileKey:
|
||||
overlap = compress_ratio == 4
|
||||
default_quant_block = 32 if use_fp4_cache else 128
|
||||
quant_block = (
|
||||
runtime_quant_block
|
||||
if runtime_quant_block is not None
|
||||
else default_quant_block
|
||||
)
|
||||
default_token_stride = head_dim // 2 if use_fp4_cache else head_dim
|
||||
token_stride = (
|
||||
runtime_token_stride
|
||||
if runtime_token_stride is not None
|
||||
else default_token_stride
|
||||
)
|
||||
default_scale_dim = head_dim // quant_block if use_fp4_cache else 4
|
||||
scale_dim = (
|
||||
runtime_scale_dim if runtime_scale_dim is not None else default_scale_dim
|
||||
)
|
||||
raw_kv_cache_block_size = cache_block_size // compress_ratio
|
||||
kv_cache_block_size = (
|
||||
raw_kv_cache_block_size if raw_kv_cache_block_size >= 1 else 1
|
||||
)
|
||||
state_block_size = (
|
||||
4
|
||||
if compress_ratio == 4
|
||||
else 8
|
||||
if compress_ratio == 128
|
||||
else cache_block_size
|
||||
)
|
||||
default_kv_block_stride = round_up(
|
||||
kv_cache_block_size * (token_stride + scale_dim),
|
||||
cache_alignment,
|
||||
)
|
||||
kv_block_stride = (
|
||||
runtime_kv_block_stride
|
||||
if runtime_kv_block_stride is not None
|
||||
else default_kv_block_stride
|
||||
)
|
||||
state_width = (
|
||||
runtime_state_width
|
||||
if runtime_state_width is not None
|
||||
else head_dim * (1 + overlap)
|
||||
)
|
||||
return self.CompileKey(
|
||||
dtype=dtype,
|
||||
use_fp4_cache=use_fp4_cache,
|
||||
head_size=head_dim,
|
||||
triton_block_size=triton.next_power_of_2(head_dim),
|
||||
state_width=state_width,
|
||||
compress_ratio=compress_ratio,
|
||||
overlap=overlap,
|
||||
rope_head_dim=rope_head_dim,
|
||||
fp8_max=448.0,
|
||||
quant_block=quant_block,
|
||||
token_stride=token_stride,
|
||||
scale_dim=scale_dim,
|
||||
block_size=state_block_size,
|
||||
kv_cache_block_size=kv_cache_block_size,
|
||||
kv_block_stride=kv_block_stride,
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
if hf_config is None:
|
||||
return []
|
||||
|
||||
cache_block_size = vllm_config.cache_config.block_size
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
dsa_indexer_uses_fp4,
|
||||
)
|
||||
|
||||
use_fp4_cache = dsa_indexer_uses_fp4(vllm_config)
|
||||
head_dim = int(getattr(hf_config, "index_head_dim", 0) or 0)
|
||||
rope_head_dim = int(getattr(hf_config, "qk_rope_head_dim", 0) or 0)
|
||||
if head_dim <= 0 or rope_head_dim <= 0 or cache_block_size <= 0:
|
||||
return []
|
||||
|
||||
cache_dtype = vllm_config.cache_config.cache_dtype
|
||||
cache_alignment = 576 if cache_dtype == "fp8_ds_mla" else 512
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
use_fp4_cache=use_fp4_cache,
|
||||
head_dim=head_dim,
|
||||
rope_head_dim=rope_head_dim,
|
||||
compress_ratio=4,
|
||||
cache_block_size=cache_block_size,
|
||||
cache_alignment=cache_alignment,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
fp32_ptr = TritonWarmupTensor(torch.float32, shape=(1, 1))
|
||||
int32_ptr = TritonWarmupTensor(torch.int32)
|
||||
int64_ptr = TritonWarmupTensor(torch.int64)
|
||||
# kv_cache supplies kv_cache_block_size (shape[1]) and KV_BLOCK_STRIDE
|
||||
# (stride(0)); reproduce both so warmup and launch specialize identically.
|
||||
kv_cache = TritonWarmupTensor(
|
||||
torch.uint8,
|
||||
shape=(1, compile_key.kv_cache_block_size, 1),
|
||||
strides=(compile_key.kv_block_stride, 1, 1),
|
||||
)
|
||||
return dict(
|
||||
state_cache=fp32_ptr,
|
||||
num_actual=1,
|
||||
token_to_req_indices=int32_ptr,
|
||||
positions=int64_ptr,
|
||||
slot_mapping=int64_ptr,
|
||||
block_table=int32_ptr,
|
||||
block_size=compile_key.block_size,
|
||||
state_width=compile_key.state_width,
|
||||
cos_sin_cache=fp32_ptr,
|
||||
kv_cache=kv_cache,
|
||||
k_cache_metadata=SimpleNamespace(slot_mapping=int64_ptr),
|
||||
pdl_kwargs={},
|
||||
head_dim=compile_key.head_size,
|
||||
rope_head_dim=compile_key.rope_head_dim,
|
||||
compress_ratio=compile_key.compress_ratio,
|
||||
overlap=compile_key.overlap,
|
||||
use_fp4_cache=compile_key.use_fp4_cache,
|
||||
rms_norm_weight=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(compile_key.head_size,),
|
||||
),
|
||||
rms_norm_eps=1e-6,
|
||||
quant_block=compile_key.quant_block,
|
||||
token_stride=compile_key.token_stride,
|
||||
scale_dim=compile_key.scale_dim,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
state_cache: torch.Tensor,
|
||||
num_actual: int,
|
||||
token_to_req_indices: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
block_size: int,
|
||||
state_width: int,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
k_cache_metadata: Any,
|
||||
pdl_kwargs: dict,
|
||||
head_dim: int,
|
||||
rope_head_dim: int,
|
||||
compress_ratio: int,
|
||||
overlap: bool,
|
||||
use_fp4_cache: bool,
|
||||
rms_norm_weight: torch.Tensor,
|
||||
rms_norm_eps: float,
|
||||
quant_block: int,
|
||||
token_stride: int,
|
||||
scale_dim: int,
|
||||
) -> LaunchSpec:
|
||||
return (num_actual,), dict(
|
||||
kernel=(
|
||||
_fused_kv_compress_norm_rope_insert_indexer_mxfp4_attn
|
||||
if use_fp4_cache
|
||||
else _fused_kv_compress_norm_rope_insert_indexer_attn
|
||||
),
|
||||
state_cache_stride0=state_cache.stride(0),
|
||||
state_cache_stride1=state_cache.stride(1),
|
||||
block_table_stride=block_table.stride(0),
|
||||
cos_sin_stride=cos_sin_cache.stride(0),
|
||||
k_cache_ptr=kv_cache,
|
||||
kv_slot_mapping_ptr=k_cache_metadata.slot_mapping,
|
||||
kv_cache_block_size=kv_cache.shape[1],
|
||||
HEAD_SIZE=head_dim,
|
||||
TRITON_BLOCK_SIZE=triton.next_power_of_2(head_dim),
|
||||
STATE_WIDTH=state_width,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
OVERLAP=overlap,
|
||||
ROPE_HEAD_DIM=rope_head_dim,
|
||||
FP8_MAX=448.0,
|
||||
QUANT_BLOCK=quant_block,
|
||||
TOKEN_STRIDE=token_stride,
|
||||
SCALE_DIM=scale_dim,
|
||||
KV_BLOCK_STRIDE=kv_cache.stride(0),
|
||||
num_warps=1,
|
||||
**pdl_kwargs,
|
||||
)
|
||||
|
||||
|
||||
_FUSED_KV_COMPRESS_NORM_ROPE_INSERT_INDEXER_TRITON_KERNEL = (
|
||||
FusedKVCompressNormRopeInsertIndexerTritonKernel()
|
||||
)
|
||||
|
||||
@@ -1,8 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
kernel_launcher,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
@@ -67,229 +76,459 @@ def _quantize_mxfp4_pair(x_lo, x_hi):
|
||||
return packed, ue8m0
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_indexer_q_rope_quant_kernel(
|
||||
pos_ptr,
|
||||
# Index Q RoPE
|
||||
index_q_ptr,
|
||||
index_q_stride0,
|
||||
index_q_stride1,
|
||||
index_q_cos_sin_ptr,
|
||||
index_q_cos_sin_stride,
|
||||
INDEX_Q_HALF_ROT_DIM: tl.constexpr,
|
||||
# Index Q Quantize
|
||||
index_q_fp8_ptr,
|
||||
index_q_fp8_stride0,
|
||||
index_q_fp8_stride1,
|
||||
INDEX_Q_HEAD_DIM: tl.constexpr,
|
||||
# Index weights
|
||||
index_weights_ptr,
|
||||
index_weights_stride,
|
||||
index_weights_softmax_scale,
|
||||
index_weights_head_scale,
|
||||
index_weights_out_ptr,
|
||||
index_weights_out_stride,
|
||||
FP8_MAX: tl.constexpr = 448.0,
|
||||
USE_FNUZ: tl.constexpr = False,
|
||||
USE_EXPLICIT_FMA: tl.constexpr = False,
|
||||
class FusedIndexerQRopeQuantTritonKernel(
|
||||
VllmTritonJitKernel["FusedIndexerQRopeQuantTritonKernel.CompileKey"]
|
||||
):
|
||||
# Layout matches the unfused reference (DeepseekV4ScalingRotaryEmbedding
|
||||
# + per_token_group_quant_fp8): GPT-J interleaved RoPE applied to the
|
||||
# LAST rope_dim dims of each head; the leading [0, NOPE_DIM) is passed
|
||||
# through unchanged.
|
||||
INDEX_Q_ROT_DIM: tl.constexpr = 2 * INDEX_Q_HALF_ROT_DIM
|
||||
INDEX_Q_NOPE_DIM: tl.constexpr = INDEX_Q_HEAD_DIM - INDEX_Q_ROT_DIM
|
||||
tl.static_assert(INDEX_Q_NOPE_DIM >= 0)
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
dtype: torch.dtype
|
||||
num_heads: int
|
||||
index_q_half_rot_dim: int
|
||||
index_q_head_dim: int
|
||||
fp8_max: float
|
||||
use_fnuz: bool
|
||||
use_explicit_fma: bool
|
||||
|
||||
tok_idx = tl.program_id(0)
|
||||
head_idx = tl.program_id(1)
|
||||
|
||||
pos = tl.load(pos_ptr + tok_idx)
|
||||
cos, sin = _get_cos_sin(
|
||||
@staticmethod
|
||||
@triton.jit
|
||||
def kernel(
|
||||
pos_ptr,
|
||||
# Index Q RoPE
|
||||
index_q_ptr,
|
||||
index_q_stride0,
|
||||
index_q_stride1,
|
||||
index_q_cos_sin_ptr,
|
||||
index_q_cos_sin_stride,
|
||||
pos,
|
||||
INDEX_Q_HALF_ROT_DIM,
|
||||
)
|
||||
half_offset = tl.arange(0, INDEX_Q_HALF_ROT_DIM)
|
||||
base_ptr = index_q_ptr + tok_idx * index_q_stride0 + head_idx * index_q_stride1
|
||||
INDEX_Q_HALF_ROT_DIM: tl.constexpr,
|
||||
# Index Q Quantize
|
||||
index_q_fp8_ptr,
|
||||
index_q_fp8_stride0,
|
||||
index_q_fp8_stride1,
|
||||
INDEX_Q_HEAD_DIM: tl.constexpr,
|
||||
# Index weights
|
||||
index_weights_ptr,
|
||||
index_weights_stride,
|
||||
index_weights_softmax_scale,
|
||||
index_weights_head_scale,
|
||||
index_weights_out_ptr,
|
||||
index_weights_out_stride,
|
||||
FP8_MAX: tl.constexpr = 448.0,
|
||||
USE_FNUZ: tl.constexpr = False,
|
||||
USE_EXPLICIT_FMA: tl.constexpr = False,
|
||||
):
|
||||
# Layout matches the unfused reference (DeepseekV4ScalingRotaryEmbedding
|
||||
# + per_token_group_quant_fp8): GPT-J interleaved RoPE applied to the
|
||||
# LAST rope_dim dims of each head; the leading [0, NOPE_DIM) is passed
|
||||
# through unchanged.
|
||||
INDEX_Q_ROT_DIM: tl.constexpr = 2 * INDEX_Q_HALF_ROT_DIM
|
||||
INDEX_Q_NOPE_DIM: tl.constexpr = INDEX_Q_HEAD_DIM - INDEX_Q_ROT_DIM
|
||||
tl.static_assert(INDEX_Q_NOPE_DIM >= 0)
|
||||
|
||||
# Interleaved (GPT-J) RoPE on dims [NOPE_DIM, HEAD_DIM):
|
||||
# even = q[NOPE_DIM + 2*i], odd = q[NOPE_DIM + 2*i + 1]
|
||||
rot_base = base_ptr + INDEX_Q_NOPE_DIM
|
||||
x_even = tl.load(rot_base + half_offset * 2).to(tl.float32)
|
||||
x_odd = tl.load(rot_base + half_offset * 2 + 1).to(tl.float32)
|
||||
if USE_EXPLICIT_FMA:
|
||||
# Match HIP rotary_embedding contraction before bf16 materialization.
|
||||
r_even = tl.fma(x_even, cos, -(x_odd * sin))
|
||||
r_odd = tl.fma(x_odd, cos, x_even * sin)
|
||||
else:
|
||||
r_even = x_even * cos - x_odd * sin
|
||||
r_odd = x_odd * cos + x_even * sin
|
||||
tok_idx = tl.program_id(0)
|
||||
head_idx = tl.program_id(1)
|
||||
|
||||
# Match reference numerics: fp32 → bf16 → fp32 before the ue8m0 absmax.
|
||||
# Same pattern as the K-side compressor kernel (fused_compress_quant_cache.py).
|
||||
r_even = r_even.to(tl.bfloat16).to(tl.float32)
|
||||
r_odd = r_odd.to(tl.bfloat16).to(tl.float32)
|
||||
|
||||
amax = tl.maximum(tl.max(tl.abs(r_even)), tl.max(tl.abs(r_odd)))
|
||||
if INDEX_Q_NOPE_DIM > 0:
|
||||
nope_offset = tl.arange(0, INDEX_Q_NOPE_DIM)
|
||||
x_nope = tl.load(base_ptr + nope_offset).to(tl.float32)
|
||||
amax = tl.maximum(amax, tl.max(tl.abs(x_nope)))
|
||||
index_q_scale = tl.div_rn(tl.maximum(amax, 1e-4), FP8_MAX)
|
||||
index_q_scale = tl.math.exp2(tl.math.ceil(tl.math.log2(index_q_scale)))
|
||||
|
||||
# Store quantized values to index_q_fp8. FNUZ (e4m3fnuz) on gfx942, OCP
|
||||
# (e4m3fn) elsewhere -- matches the K cache.
|
||||
fp8_dtype = tl.float8e4b8 if USE_FNUZ else tl.float8e4nv
|
||||
fp8_base_ptr = (
|
||||
index_q_fp8_ptr + tok_idx * index_q_fp8_stride0 + head_idx * index_q_fp8_stride1
|
||||
)
|
||||
if INDEX_Q_NOPE_DIM > 0:
|
||||
tl.store(
|
||||
fp8_base_ptr + nope_offset,
|
||||
tl.div_rn(x_nope, index_q_scale).to(fp8_dtype),
|
||||
pos = tl.load(pos_ptr + tok_idx)
|
||||
cos, sin = _get_cos_sin(
|
||||
index_q_cos_sin_ptr,
|
||||
index_q_cos_sin_stride,
|
||||
pos,
|
||||
INDEX_Q_HALF_ROT_DIM,
|
||||
)
|
||||
fp8_rot_base = fp8_base_ptr + INDEX_Q_NOPE_DIM
|
||||
tl.store(
|
||||
fp8_rot_base + half_offset * 2,
|
||||
tl.div_rn(r_even, index_q_scale).to(fp8_dtype),
|
||||
)
|
||||
tl.store(
|
||||
fp8_rot_base + half_offset * 2 + 1,
|
||||
tl.div_rn(r_odd, index_q_scale).to(fp8_dtype),
|
||||
)
|
||||
half_offset = tl.arange(0, INDEX_Q_HALF_ROT_DIM)
|
||||
base_ptr = index_q_ptr + tok_idx * index_q_stride0 + head_idx * index_q_stride1
|
||||
|
||||
# FP8 weight-fold contract:
|
||||
# index_weights_out = index_weights * q_scale * softmax_scale * head_scale
|
||||
# The per-token-per-head q_scale (fp32) IS folded into the output weights
|
||||
# here because FP8 Q is stored WITHOUT a companion scale tensor — the
|
||||
# downstream fp8_fp4_mqa_logits/fp8_fp4_paged_mqa_logits kernels use `weights` to
|
||||
# apply per-token Q scale inline. See the MXFP4 kernel below for the
|
||||
# contrasting convention (scales live with the Q values, weights are NOT
|
||||
# q-scaled).
|
||||
index_weights = tl.load(
|
||||
index_weights_ptr + tok_idx * index_weights_stride + head_idx
|
||||
)
|
||||
index_weights = index_weights.to(tl.float32)
|
||||
index_weights *= index_q_scale
|
||||
index_weights *= index_weights_softmax_scale
|
||||
index_weights *= index_weights_head_scale
|
||||
tl.store(
|
||||
index_weights_out_ptr + tok_idx * index_weights_out_stride + head_idx,
|
||||
index_weights,
|
||||
)
|
||||
# Interleaved (GPT-J) RoPE on dims [NOPE_DIM, HEAD_DIM):
|
||||
# even = q[NOPE_DIM + 2*i], odd = q[NOPE_DIM + 2*i + 1]
|
||||
rot_base = base_ptr + INDEX_Q_NOPE_DIM
|
||||
x_even = tl.load(rot_base + half_offset * 2).to(tl.float32)
|
||||
x_odd = tl.load(rot_base + half_offset * 2 + 1).to(tl.float32)
|
||||
if USE_EXPLICIT_FMA:
|
||||
# Match HIP rotary_embedding contraction before bf16 materialization.
|
||||
r_even = tl.fma(x_even, cos, -(x_odd * sin))
|
||||
r_odd = tl.fma(x_odd, cos, x_even * sin)
|
||||
else:
|
||||
r_even = x_even * cos - x_odd * sin
|
||||
r_odd = x_odd * cos + x_even * sin
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_indexer_q_rope_mxfp4_kernel(
|
||||
pos_ptr,
|
||||
# Index Q RoPE input (fp/bf16)
|
||||
index_q_ptr,
|
||||
index_q_stride0,
|
||||
index_q_stride1,
|
||||
index_q_cos_sin_ptr,
|
||||
index_q_cos_sin_stride,
|
||||
INDEX_Q_HALF_ROT_DIM: tl.constexpr,
|
||||
# MXFP4 Q outputs
|
||||
index_q_mxfp4_ptr, # uint8, (T, H, HEAD_DIM // 2)
|
||||
index_q_mxfp4_stride0,
|
||||
index_q_mxfp4_stride1,
|
||||
index_q_scale_ptr, # uint8 ue8m0, (T, H, HEAD_DIM // BLOCK)
|
||||
index_q_scale_stride0,
|
||||
index_q_scale_stride1,
|
||||
INDEX_Q_HEAD_DIM: tl.constexpr,
|
||||
MXFP4_BLOCK: tl.constexpr,
|
||||
# Weights (NO per-token q_scale fold for MXFP4; per-block scales stay
|
||||
# with the Q values in the output scale tensor).
|
||||
index_weights_ptr,
|
||||
index_weights_stride,
|
||||
index_weights_softmax_scale,
|
||||
index_weights_head_scale,
|
||||
index_weights_out_ptr,
|
||||
index_weights_out_stride,
|
||||
):
|
||||
INDEX_Q_ROT_DIM: tl.constexpr = 2 * INDEX_Q_HALF_ROT_DIM
|
||||
INDEX_Q_NOPE_DIM: tl.constexpr = INDEX_Q_HEAD_DIM - INDEX_Q_ROT_DIM
|
||||
NUM_NOPE_BLOCKS: tl.constexpr = INDEX_Q_NOPE_DIM // MXFP4_BLOCK
|
||||
NUM_ROPE_BLOCKS: tl.constexpr = INDEX_Q_ROT_DIM // MXFP4_BLOCK
|
||||
HALF_BLOCK: tl.constexpr = MXFP4_BLOCK // 2
|
||||
tl.static_assert(INDEX_Q_NOPE_DIM >= 0)
|
||||
tl.static_assert(INDEX_Q_NOPE_DIM % MXFP4_BLOCK == 0)
|
||||
tl.static_assert(INDEX_Q_ROT_DIM % MXFP4_BLOCK == 0)
|
||||
tl.static_assert(MXFP4_BLOCK % 2 == 0)
|
||||
|
||||
tok_idx = tl.program_id(0)
|
||||
head_idx = tl.program_id(1)
|
||||
|
||||
pos = tl.load(pos_ptr + tok_idx)
|
||||
|
||||
q_base = index_q_ptr + tok_idx * index_q_stride0 + head_idx * index_q_stride1
|
||||
out_base = (
|
||||
index_q_mxfp4_ptr
|
||||
+ tok_idx * index_q_mxfp4_stride0
|
||||
+ head_idx * index_q_mxfp4_stride1
|
||||
)
|
||||
scale_base = (
|
||||
index_q_scale_ptr
|
||||
+ tok_idx * index_q_scale_stride0
|
||||
+ head_idx * index_q_scale_stride1
|
||||
)
|
||||
|
||||
half_off = tl.arange(0, HALF_BLOCK)
|
||||
|
||||
# ---- NoPE blocks: direct load, pair as (even-index, odd-index) values ----
|
||||
for b in tl.static_range(NUM_NOPE_BLOCKS):
|
||||
base = b * MXFP4_BLOCK
|
||||
x_lo = tl.load(q_base + base + half_off * 2).to(tl.float32)
|
||||
x_hi = tl.load(q_base + base + half_off * 2 + 1).to(tl.float32)
|
||||
packed, ue8m0 = _quantize_mxfp4_pair(x_lo, x_hi)
|
||||
tl.store(out_base + base // 2 + half_off, packed)
|
||||
tl.store(scale_base + b, ue8m0)
|
||||
|
||||
# ---- RoPE blocks: apply GPT-J interleaved RoPE to the block's 16 pairs,
|
||||
# then quantize. Each block covers HALF_BLOCK (=16) cos/sin pairs. ----
|
||||
rot_q_base = q_base + INDEX_Q_NOPE_DIM
|
||||
for b in tl.static_range(NUM_ROPE_BLOCKS):
|
||||
pair_off = b * HALF_BLOCK + half_off # indices in [0, HALF_ROT_DIM)
|
||||
cos_b = tl.load(
|
||||
index_q_cos_sin_ptr + pos * index_q_cos_sin_stride + pair_off
|
||||
).to(tl.float32)
|
||||
sin_b = tl.load(
|
||||
index_q_cos_sin_ptr
|
||||
+ pos * index_q_cos_sin_stride
|
||||
+ pair_off
|
||||
+ INDEX_Q_HALF_ROT_DIM
|
||||
).to(tl.float32)
|
||||
x_even = tl.load(rot_q_base + pair_off * 2).to(tl.float32)
|
||||
x_odd = tl.load(rot_q_base + pair_off * 2 + 1).to(tl.float32)
|
||||
r_even = x_even * cos_b - x_odd * sin_b
|
||||
r_odd = x_odd * cos_b + x_even * sin_b
|
||||
# bf16 roundtrip for parity with the FP8 kernel / reference numerics.
|
||||
# Match reference numerics: fp32 → bf16 → fp32 before the ue8m0 absmax.
|
||||
# Same pattern as the K-side compressor kernel (fused_compress_quant_cache.py).
|
||||
r_even = r_even.to(tl.bfloat16).to(tl.float32)
|
||||
r_odd = r_odd.to(tl.bfloat16).to(tl.float32)
|
||||
packed, ue8m0 = _quantize_mxfp4_pair(r_even, r_odd)
|
||||
rope_byte_off = (INDEX_Q_NOPE_DIM + b * MXFP4_BLOCK) // 2
|
||||
tl.store(out_base + rope_byte_off + half_off, packed)
|
||||
tl.store(scale_base + NUM_NOPE_BLOCKS + b, ue8m0)
|
||||
|
||||
# MXFP4 weight-fold contract:
|
||||
# index_weights_out = index_weights * softmax_scale * head_scale
|
||||
# NOTE: q_scale is NOT folded here (contrast with the FP8 kernel above).
|
||||
# MXFP4 Q emits a separate ue8m0 scale tensor of shape
|
||||
# (T, H, HEAD_DIM // MXFP4_BLOCK) alongside the packed values, so each
|
||||
# per-block scale is applied by the downstream MXFP4 logits kernel when
|
||||
# dequantizing Q — there is no per-token scalar to fold into `weights`.
|
||||
index_weights = tl.load(
|
||||
index_weights_ptr + tok_idx * index_weights_stride + head_idx
|
||||
).to(tl.float32)
|
||||
index_weights *= index_weights_softmax_scale
|
||||
index_weights *= index_weights_head_scale
|
||||
tl.store(
|
||||
index_weights_out_ptr + tok_idx * index_weights_out_stride + head_idx,
|
||||
index_weights,
|
||||
)
|
||||
amax = tl.maximum(tl.max(tl.abs(r_even)), tl.max(tl.abs(r_odd)))
|
||||
if INDEX_Q_NOPE_DIM > 0:
|
||||
nope_offset = tl.arange(0, INDEX_Q_NOPE_DIM)
|
||||
x_nope = tl.load(base_ptr + nope_offset).to(tl.float32)
|
||||
amax = tl.maximum(amax, tl.max(tl.abs(x_nope)))
|
||||
index_q_scale = tl.div_rn(tl.maximum(amax, 1e-4), FP8_MAX)
|
||||
index_q_scale = tl.math.exp2(tl.math.ceil(tl.math.log2(index_q_scale)))
|
||||
|
||||
# Store quantized values to index_q_fp8. FNUZ (e4m3fnuz) on gfx942, OCP
|
||||
# (e4m3fn) elsewhere -- matches the K cache.
|
||||
fp8_dtype = tl.float8e4b8 if USE_FNUZ else tl.float8e4nv
|
||||
fp8_base_ptr = (
|
||||
index_q_fp8_ptr
|
||||
+ tok_idx * index_q_fp8_stride0
|
||||
+ head_idx * index_q_fp8_stride1
|
||||
)
|
||||
if INDEX_Q_NOPE_DIM > 0:
|
||||
tl.store(
|
||||
fp8_base_ptr + nope_offset,
|
||||
tl.div_rn(x_nope, index_q_scale).to(fp8_dtype),
|
||||
)
|
||||
fp8_rot_base = fp8_base_ptr + INDEX_Q_NOPE_DIM
|
||||
tl.store(
|
||||
fp8_rot_base + half_offset * 2,
|
||||
tl.div_rn(r_even, index_q_scale).to(fp8_dtype),
|
||||
)
|
||||
tl.store(
|
||||
fp8_rot_base + half_offset * 2 + 1,
|
||||
tl.div_rn(r_odd, index_q_scale).to(fp8_dtype),
|
||||
)
|
||||
|
||||
# FP8 weight-fold contract:
|
||||
# index_weights_out = index_weights * q_scale * softmax_scale * head_scale
|
||||
# The per-token-per-head q_scale (fp32) IS folded into the output weights
|
||||
# here because FP8 Q is stored WITHOUT a companion scale tensor — the
|
||||
# downstream fp8_fp4_mqa_logits/fp8_fp4_paged_mqa_logits kernels use
|
||||
# `weights` to apply per-token Q scale inline. See the MXFP4 kernel below for
|
||||
# contrasting convention (scales live with the Q values, weights are NOT
|
||||
# q-scaled).
|
||||
index_weights = tl.load(
|
||||
index_weights_ptr + tok_idx * index_weights_stride + head_idx
|
||||
)
|
||||
index_weights = index_weights.to(tl.float32)
|
||||
index_weights *= index_q_scale
|
||||
index_weights *= index_weights_softmax_scale
|
||||
index_weights *= index_weights_head_scale
|
||||
tl.store(
|
||||
index_weights_out_ptr + tok_idx * index_weights_out_stride + head_idx,
|
||||
index_weights,
|
||||
)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
rope_dim: int,
|
||||
use_fnuz: bool,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
dtype=dtype,
|
||||
num_heads=num_heads,
|
||||
index_q_half_rot_dim=rope_dim // 2,
|
||||
index_q_head_dim=head_dim,
|
||||
fp8_max=224.0 if use_fnuz else 448.0,
|
||||
use_fnuz=use_fnuz,
|
||||
use_explicit_fma=current_platform.is_rocm(),
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
num_heads = int(getattr(hf_config, "index_n_heads", 0) or 0)
|
||||
head_dim = int(getattr(hf_config, "index_head_dim", 0) or 0)
|
||||
rope_dim = int(getattr(hf_config, "qk_rope_head_dim", 0) or 0)
|
||||
if num_heads <= 0 or head_dim <= 0 or rope_dim <= 0:
|
||||
return []
|
||||
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
num_heads=num_heads,
|
||||
head_dim=head_dim,
|
||||
rope_dim=rope_dim,
|
||||
use_fnuz=current_platform.fp8_dtype() == torch.float8_e4m3fnuz,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
q_stride0 = compile_key.num_heads * compile_key.index_q_head_dim
|
||||
return dict(
|
||||
positions=TritonWarmupTensor(torch.int64),
|
||||
index_q=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.num_heads, compile_key.index_q_head_dim),
|
||||
strides=(q_stride0, compile_key.index_q_head_dim, 1),
|
||||
),
|
||||
index_q_cos_sin_cache=TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, compile_key.index_q_half_rot_dim * 2),
|
||||
),
|
||||
index_weights=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.num_heads),
|
||||
),
|
||||
index_weights_softmax_scale=1.0,
|
||||
index_weights_head_scale=1.0,
|
||||
index_q_fp8=TritonWarmupTensor(
|
||||
current_platform.fp8_dtype(),
|
||||
shape=(1, compile_key.num_heads, compile_key.index_q_head_dim),
|
||||
strides=(q_stride0, compile_key.index_q_head_dim, 1),
|
||||
),
|
||||
index_weights_out=TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, compile_key.num_heads),
|
||||
),
|
||||
fp8_max=compile_key.fp8_max,
|
||||
use_fnuz=compile_key.use_fnuz,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
index_q: torch.Tensor,
|
||||
index_q_cos_sin_cache: torch.Tensor,
|
||||
index_weights: torch.Tensor,
|
||||
index_weights_softmax_scale: float,
|
||||
index_weights_head_scale: float,
|
||||
index_q_fp8: torch.Tensor,
|
||||
index_weights_out: torch.Tensor,
|
||||
*,
|
||||
fp8_max: float,
|
||||
use_fnuz: bool,
|
||||
) -> LaunchSpec:
|
||||
num_tokens = positions.shape[0]
|
||||
num_index_q_heads = index_q.shape[1]
|
||||
return (num_tokens, num_index_q_heads), dict(
|
||||
pos_ptr=positions,
|
||||
index_q_stride0=index_q.stride(0),
|
||||
index_q_stride1=index_q.stride(1),
|
||||
index_q_cos_sin_ptr=index_q_cos_sin_cache,
|
||||
index_q_cos_sin_stride=index_q_cos_sin_cache.stride(0),
|
||||
INDEX_Q_HALF_ROT_DIM=index_q_cos_sin_cache.shape[-1] // 2,
|
||||
index_q_fp8_stride0=index_q_fp8.stride(0),
|
||||
index_q_fp8_stride1=index_q_fp8.stride(1),
|
||||
INDEX_Q_HEAD_DIM=index_q.shape[2],
|
||||
index_weights_stride=index_weights.stride(0),
|
||||
index_weights_out_stride=index_weights_out.stride(0),
|
||||
FP8_MAX=fp8_max,
|
||||
USE_FNUZ=use_fnuz,
|
||||
USE_EXPLICIT_FMA=current_platform.is_rocm(),
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
|
||||
class FusedIndexerQRopeMxFp4TritonKernel(
|
||||
VllmTritonJitKernel["FusedIndexerQRopeMxFp4TritonKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
dtype: torch.dtype
|
||||
num_heads: int
|
||||
index_q_half_rot_dim: int
|
||||
index_q_head_dim: int
|
||||
mxfp4_block: int
|
||||
|
||||
@staticmethod
|
||||
@triton.jit
|
||||
def kernel(
|
||||
pos_ptr,
|
||||
# Index Q RoPE input (fp/bf16)
|
||||
index_q_ptr,
|
||||
index_q_stride0,
|
||||
index_q_stride1,
|
||||
index_q_cos_sin_ptr,
|
||||
index_q_cos_sin_stride,
|
||||
INDEX_Q_HALF_ROT_DIM: tl.constexpr,
|
||||
# MXFP4 Q outputs
|
||||
index_q_mxfp4_ptr, # uint8, (T, H, HEAD_DIM // 2)
|
||||
index_q_mxfp4_stride0,
|
||||
index_q_mxfp4_stride1,
|
||||
index_q_scale_ptr, # uint8 ue8m0, (T, H, HEAD_DIM // BLOCK)
|
||||
index_q_scale_stride0,
|
||||
index_q_scale_stride1,
|
||||
INDEX_Q_HEAD_DIM: tl.constexpr,
|
||||
MXFP4_BLOCK: tl.constexpr,
|
||||
# Weights (NO per-token q_scale fold for MXFP4; per-block scales stay
|
||||
# with the Q values in the output scale tensor).
|
||||
index_weights_ptr,
|
||||
index_weights_stride,
|
||||
index_weights_softmax_scale,
|
||||
index_weights_head_scale,
|
||||
index_weights_out_ptr,
|
||||
index_weights_out_stride,
|
||||
):
|
||||
INDEX_Q_ROT_DIM: tl.constexpr = 2 * INDEX_Q_HALF_ROT_DIM
|
||||
INDEX_Q_NOPE_DIM: tl.constexpr = INDEX_Q_HEAD_DIM - INDEX_Q_ROT_DIM
|
||||
NUM_NOPE_BLOCKS: tl.constexpr = INDEX_Q_NOPE_DIM // MXFP4_BLOCK
|
||||
NUM_ROPE_BLOCKS: tl.constexpr = INDEX_Q_ROT_DIM // MXFP4_BLOCK
|
||||
HALF_BLOCK: tl.constexpr = MXFP4_BLOCK // 2
|
||||
tl.static_assert(INDEX_Q_NOPE_DIM >= 0)
|
||||
tl.static_assert(INDEX_Q_NOPE_DIM % MXFP4_BLOCK == 0)
|
||||
tl.static_assert(INDEX_Q_ROT_DIM % MXFP4_BLOCK == 0)
|
||||
tl.static_assert(MXFP4_BLOCK % 2 == 0)
|
||||
|
||||
tok_idx = tl.program_id(0)
|
||||
head_idx = tl.program_id(1)
|
||||
|
||||
pos = tl.load(pos_ptr + tok_idx)
|
||||
|
||||
q_base = index_q_ptr + tok_idx * index_q_stride0 + head_idx * index_q_stride1
|
||||
out_base = (
|
||||
index_q_mxfp4_ptr
|
||||
+ tok_idx * index_q_mxfp4_stride0
|
||||
+ head_idx * index_q_mxfp4_stride1
|
||||
)
|
||||
scale_base = (
|
||||
index_q_scale_ptr
|
||||
+ tok_idx * index_q_scale_stride0
|
||||
+ head_idx * index_q_scale_stride1
|
||||
)
|
||||
|
||||
half_off = tl.arange(0, HALF_BLOCK)
|
||||
|
||||
# ---- NoPE blocks: direct load, pair as (even-index, odd-index) values ----
|
||||
for b in tl.static_range(NUM_NOPE_BLOCKS):
|
||||
base = b * MXFP4_BLOCK
|
||||
x_lo = tl.load(q_base + base + half_off * 2).to(tl.float32)
|
||||
x_hi = tl.load(q_base + base + half_off * 2 + 1).to(tl.float32)
|
||||
packed, ue8m0 = _quantize_mxfp4_pair(x_lo, x_hi)
|
||||
tl.store(out_base + base // 2 + half_off, packed)
|
||||
tl.store(scale_base + b, ue8m0)
|
||||
|
||||
# ---- RoPE blocks: apply GPT-J interleaved RoPE to the block's 16 pairs,
|
||||
# then quantize. Each block covers HALF_BLOCK (=16) cos/sin pairs. ----
|
||||
rot_q_base = q_base + INDEX_Q_NOPE_DIM
|
||||
for b in tl.static_range(NUM_ROPE_BLOCKS):
|
||||
pair_off = b * HALF_BLOCK + half_off # indices in [0, HALF_ROT_DIM)
|
||||
cos_b = tl.load(
|
||||
index_q_cos_sin_ptr + pos * index_q_cos_sin_stride + pair_off
|
||||
).to(tl.float32)
|
||||
sin_b = tl.load(
|
||||
index_q_cos_sin_ptr
|
||||
+ pos * index_q_cos_sin_stride
|
||||
+ pair_off
|
||||
+ INDEX_Q_HALF_ROT_DIM
|
||||
).to(tl.float32)
|
||||
x_even = tl.load(rot_q_base + pair_off * 2).to(tl.float32)
|
||||
x_odd = tl.load(rot_q_base + pair_off * 2 + 1).to(tl.float32)
|
||||
r_even = x_even * cos_b - x_odd * sin_b
|
||||
r_odd = x_odd * cos_b + x_even * sin_b
|
||||
# bf16 roundtrip for parity with the FP8 kernel / reference numerics.
|
||||
r_even = r_even.to(tl.bfloat16).to(tl.float32)
|
||||
r_odd = r_odd.to(tl.bfloat16).to(tl.float32)
|
||||
packed, ue8m0 = _quantize_mxfp4_pair(r_even, r_odd)
|
||||
rope_byte_off = (INDEX_Q_NOPE_DIM + b * MXFP4_BLOCK) // 2
|
||||
tl.store(out_base + rope_byte_off + half_off, packed)
|
||||
tl.store(scale_base + NUM_NOPE_BLOCKS + b, ue8m0)
|
||||
|
||||
# MXFP4 weight-fold contract:
|
||||
# index_weights_out = index_weights * softmax_scale * head_scale
|
||||
# NOTE: q_scale is NOT folded here (contrast with the FP8 kernel above).
|
||||
# MXFP4 Q emits a separate ue8m0 scale tensor of shape
|
||||
# (T, H, HEAD_DIM // MXFP4_BLOCK) alongside the packed values, so each
|
||||
# per-block scale is applied by the downstream MXFP4 logits kernel when
|
||||
# dequantizing Q — there is no per-token scalar to fold into `weights`.
|
||||
index_weights = tl.load(
|
||||
index_weights_ptr + tok_idx * index_weights_stride + head_idx
|
||||
).to(tl.float32)
|
||||
index_weights *= index_weights_softmax_scale
|
||||
index_weights *= index_weights_head_scale
|
||||
tl.store(
|
||||
index_weights_out_ptr + tok_idx * index_weights_out_stride + head_idx,
|
||||
index_weights,
|
||||
)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
rope_dim: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
dtype=dtype,
|
||||
num_heads=num_heads,
|
||||
index_q_half_rot_dim=rope_dim // 2,
|
||||
index_q_head_dim=head_dim,
|
||||
mxfp4_block=MXFP4_BLOCK_SIZE,
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
num_heads = int(getattr(hf_config, "index_n_heads", 0) or 0)
|
||||
head_dim = int(getattr(hf_config, "index_head_dim", 0) or 0)
|
||||
rope_dim = int(getattr(hf_config, "qk_rope_head_dim", 0) or 0)
|
||||
if num_heads <= 0 or head_dim <= 0 or rope_dim <= 0:
|
||||
return []
|
||||
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
num_heads=num_heads,
|
||||
head_dim=head_dim,
|
||||
rope_dim=rope_dim,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
q_stride0 = compile_key.num_heads * compile_key.index_q_head_dim
|
||||
packed_head_dim = compile_key.index_q_head_dim // 2
|
||||
scale_head_dim = compile_key.index_q_head_dim // compile_key.mxfp4_block
|
||||
return dict(
|
||||
positions=TritonWarmupTensor(torch.int64),
|
||||
index_q=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.num_heads, compile_key.index_q_head_dim),
|
||||
strides=(q_stride0, compile_key.index_q_head_dim, 1),
|
||||
),
|
||||
index_q_cos_sin_cache=TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, compile_key.index_q_half_rot_dim * 2),
|
||||
),
|
||||
index_weights=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.num_heads),
|
||||
),
|
||||
index_weights_softmax_scale=1.0,
|
||||
index_weights_head_scale=1.0,
|
||||
index_q_packed=TritonWarmupTensor(
|
||||
torch.uint8,
|
||||
shape=(1, compile_key.num_heads, packed_head_dim),
|
||||
),
|
||||
index_q_scale=TritonWarmupTensor(
|
||||
torch.uint8,
|
||||
shape=(1, compile_key.num_heads, scale_head_dim),
|
||||
),
|
||||
index_weights_out=TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, compile_key.num_heads),
|
||||
),
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
index_q: torch.Tensor,
|
||||
index_q_cos_sin_cache: torch.Tensor,
|
||||
index_weights: torch.Tensor,
|
||||
index_weights_softmax_scale: float,
|
||||
index_weights_head_scale: float,
|
||||
index_q_packed: torch.Tensor,
|
||||
index_q_scale: torch.Tensor,
|
||||
index_weights_out: torch.Tensor,
|
||||
) -> LaunchSpec:
|
||||
num_tokens = positions.shape[0]
|
||||
num_index_q_heads = index_q.shape[1]
|
||||
return (num_tokens, num_index_q_heads), dict(
|
||||
pos_ptr=positions,
|
||||
index_q_stride0=index_q.stride(0),
|
||||
index_q_stride1=index_q.stride(1),
|
||||
index_q_cos_sin_ptr=index_q_cos_sin_cache,
|
||||
index_q_cos_sin_stride=index_q_cos_sin_cache.stride(0),
|
||||
INDEX_Q_HALF_ROT_DIM=index_q_cos_sin_cache.shape[-1] // 2,
|
||||
index_q_mxfp4_ptr=index_q_packed,
|
||||
index_q_mxfp4_stride0=index_q_packed.stride(0),
|
||||
index_q_mxfp4_stride1=index_q_packed.stride(1),
|
||||
index_q_scale_stride0=index_q_scale.stride(0),
|
||||
index_q_scale_stride1=index_q_scale.stride(1),
|
||||
INDEX_Q_HEAD_DIM=index_q.shape[2],
|
||||
MXFP4_BLOCK=MXFP4_BLOCK_SIZE,
|
||||
index_weights_stride=index_weights.stride(0),
|
||||
index_weights_out_stride=index_weights_out.stride(0),
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
|
||||
def fused_indexer_q_rope_quant(
|
||||
@@ -386,29 +625,16 @@ def fused_indexer_q_rope_quant(
|
||||
index_weights_out,
|
||||
)
|
||||
else:
|
||||
_fused_indexer_q_rope_mxfp4_kernel[(num_tokens, num_index_q_heads)](
|
||||
_FUSED_INDEXER_Q_ROPE_MXFP4_TRITON_KERNEL(
|
||||
positions,
|
||||
index_q,
|
||||
index_q.stride(0),
|
||||
index_q.stride(1),
|
||||
index_q_cos_sin_cache,
|
||||
index_q_cos_sin_cache.stride(0),
|
||||
index_q_cos_sin_cache.shape[-1] // 2,
|
||||
index_q_packed,
|
||||
index_q_packed.stride(0),
|
||||
index_q_packed.stride(1),
|
||||
index_q_scale,
|
||||
index_q_scale.stride(0),
|
||||
index_q_scale.stride(1),
|
||||
index_q_head_dim,
|
||||
MXFP4_BLOCK_SIZE,
|
||||
index_weights,
|
||||
index_weights.stride(0),
|
||||
index_weights_softmax_scale,
|
||||
index_weights_head_scale,
|
||||
index_q_packed,
|
||||
index_q_scale,
|
||||
index_weights_out,
|
||||
index_weights_out.stride(0),
|
||||
num_warps=1, # TODO: Tune this
|
||||
)
|
||||
|
||||
# Values stay uint8 (2 E2M1 nibbles per byte). Scales are 4 ue8m0
|
||||
@@ -453,27 +679,20 @@ def fused_indexer_q_rope_quant(
|
||||
index_weights_out,
|
||||
)
|
||||
else:
|
||||
_fused_indexer_q_rope_quant_kernel[(num_tokens, num_index_q_heads)](
|
||||
_FUSED_INDEXER_Q_ROPE_QUANT_TRITON_KERNEL(
|
||||
positions,
|
||||
index_q,
|
||||
index_q.stride(0),
|
||||
index_q.stride(1),
|
||||
index_q_cos_sin_cache,
|
||||
index_q_cos_sin_cache.stride(0),
|
||||
index_q_cos_sin_cache.shape[-1] // 2,
|
||||
index_q_fp8,
|
||||
index_q_fp8.stride(0),
|
||||
index_q_fp8.stride(1),
|
||||
index_q_head_dim,
|
||||
index_weights,
|
||||
index_weights.stride(0),
|
||||
index_weights_softmax_scale,
|
||||
index_weights_head_scale,
|
||||
index_q_fp8,
|
||||
index_weights_out,
|
||||
index_weights_out.stride(0),
|
||||
FP8_MAX=fp8_max,
|
||||
USE_FNUZ=use_fnuz,
|
||||
USE_EXPLICIT_FMA=current_platform.is_rocm(),
|
||||
num_warps=1, # TODO: Tune this
|
||||
fp8_max=fp8_max,
|
||||
use_fnuz=use_fnuz,
|
||||
)
|
||||
return index_q_fp8, index_weights_out
|
||||
|
||||
|
||||
_FUSED_INDEXER_Q_ROPE_QUANT_TRITON_KERNEL = FusedIndexerQRopeQuantTritonKernel()
|
||||
_FUSED_INDEXER_Q_ROPE_MXFP4_TRITON_KERNEL = FusedIndexerQRopeMxFp4TritonKernel()
|
||||
|
||||
@@ -7,145 +7,327 @@ Output scale format is pre-transformed (MN-major TMA-aligned; FP32 on SM90,
|
||||
INT32-packed UE8M0 on SM100) so fp8_einsum skips transform_sf_into_required_layout.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
kernel_launcher,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["num_tokens"])
|
||||
def _fused_inv_rope_fp8_quant_per_head(
|
||||
o_ptr,
|
||||
positions_ptr,
|
||||
cos_sin_cache_ptr,
|
||||
fp8_ptr,
|
||||
scale_ptr,
|
||||
num_tokens,
|
||||
heads_per_group: tl.constexpr,
|
||||
o_stride_token,
|
||||
o_stride_head,
|
||||
cache_stride_pos,
|
||||
fp8_stride_group,
|
||||
fp8_stride_token,
|
||||
scale_stride_group,
|
||||
scale_stride_k,
|
||||
fp8_max: tl.constexpr,
|
||||
eps: tl.constexpr,
|
||||
QUANT_GROUP_SIZE: tl.constexpr,
|
||||
CHUNKS_PER_HEAD: tl.constexpr,
|
||||
ROPE_START: tl.constexpr,
|
||||
HALF_ROPE: tl.constexpr,
|
||||
TMA_ALIGNED_SCALES: tl.constexpr,
|
||||
USE_GDC: tl.constexpr,
|
||||
launch_pdl: tl.constexpr, # triton metadata
|
||||
class FusedInvRopeFP8QuantKernel(
|
||||
VllmTritonJitKernel["FusedInvRopeFP8QuantKernel.CompileKey"]
|
||||
):
|
||||
# Cast every stride to int64 — without this, Python-int strides are
|
||||
# inferred as int32 and `pid_token(int64) × stride(int32)` can lower to
|
||||
# int32 arithmetic, wrapping past 2³¹ for large prefill batches → IMA.
|
||||
pid_token = tl.program_id(0).to(tl.int64)
|
||||
pid_gh = tl.program_id(1).to(tl.int64)
|
||||
o_stride_token = o_stride_token.to(tl.int64)
|
||||
o_stride_head = o_stride_head.to(tl.int64)
|
||||
cache_stride_pos = cache_stride_pos.to(tl.int64)
|
||||
fp8_stride_group = fp8_stride_group.to(tl.int64)
|
||||
fp8_stride_token = fp8_stride_token.to(tl.int64)
|
||||
scale_stride_group = scale_stride_group.to(tl.int64)
|
||||
scale_stride_k = scale_stride_k.to(tl.int64)
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
heads_per_group: int
|
||||
fp8_max: float
|
||||
quant_group_size: int
|
||||
chunks_per_head: int
|
||||
rope_start: int
|
||||
half_rope: int
|
||||
tma_aligned_scales: bool
|
||||
launch_pdl: bool
|
||||
|
||||
g = pid_gh // heads_per_group
|
||||
head_in_group = pid_gh % heads_per_group
|
||||
global_head = pid_gh
|
||||
qb_start = head_in_group * CHUNKS_PER_HEAD
|
||||
if USE_GDC:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.extra.cuda.gdc_wait()
|
||||
# Padding rows in the TMA-aligned scale buffer: fill with zero and skip quant.
|
||||
if pid_token >= num_tokens:
|
||||
@staticmethod
|
||||
# scale_stride_k = align(num_tokens, 4) is only 4-aligned, so its Triton int
|
||||
# class varies with batch size; not specialized so one warmup key covers all
|
||||
# (int64-cast, scalar addressing only, so perf-neutral).
|
||||
@triton.jit(do_not_specialize=["num_tokens", "scale_stride_k"])
|
||||
def kernel(
|
||||
o_ptr,
|
||||
positions_ptr,
|
||||
cos_sin_cache_ptr,
|
||||
fp8_ptr,
|
||||
scale_ptr,
|
||||
num_tokens,
|
||||
heads_per_group: tl.constexpr,
|
||||
o_stride_token,
|
||||
o_stride_head,
|
||||
cache_stride_pos,
|
||||
fp8_stride_group,
|
||||
fp8_stride_token,
|
||||
scale_stride_group,
|
||||
scale_stride_k,
|
||||
fp8_max: tl.constexpr,
|
||||
eps: tl.constexpr,
|
||||
QUANT_GROUP_SIZE: tl.constexpr,
|
||||
CHUNKS_PER_HEAD: tl.constexpr,
|
||||
ROPE_START: tl.constexpr,
|
||||
HALF_ROPE: tl.constexpr,
|
||||
TMA_ALIGNED_SCALES: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
# Cast every stride to int64 — without this, Python-int strides are
|
||||
# inferred as int32 and `pid_token(int64) × stride(int32)` can lower to
|
||||
# int32 arithmetic, wrapping past 2³¹ for large prefill batches → IMA.
|
||||
pid_token = tl.program_id(0).to(tl.int64)
|
||||
pid_gh = tl.program_id(1).to(tl.int64)
|
||||
o_stride_token = o_stride_token.to(tl.int64)
|
||||
o_stride_head = o_stride_head.to(tl.int64)
|
||||
cache_stride_pos = cache_stride_pos.to(tl.int64)
|
||||
fp8_stride_group = fp8_stride_group.to(tl.int64)
|
||||
fp8_stride_token = fp8_stride_token.to(tl.int64)
|
||||
scale_stride_group = scale_stride_group.to(tl.int64)
|
||||
scale_stride_k = scale_stride_k.to(tl.int64)
|
||||
|
||||
g = pid_gh // heads_per_group
|
||||
head_in_group = pid_gh % heads_per_group
|
||||
global_head = pid_gh
|
||||
qb_start = head_in_group * CHUNKS_PER_HEAD
|
||||
if launch_pdl:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
tl.extra.cuda.gdc_wait()
|
||||
# Padding rows in the TMA-aligned scale buffer: fill with zero and skip quant.
|
||||
if pid_token >= num_tokens:
|
||||
if TMA_ALIGNED_SCALES:
|
||||
scale_addr = (
|
||||
scale_ptr
|
||||
+ g * scale_stride_group
|
||||
+ pid_token
|
||||
+ head_in_group * scale_stride_k
|
||||
)
|
||||
tl.store(scale_addr, tl.zeros((), dtype=tl.int32))
|
||||
else:
|
||||
block_offsets = tl.arange(0, CHUNKS_PER_HEAD)
|
||||
qb_indices = qb_start + block_offsets
|
||||
scale_addrs = (
|
||||
scale_ptr
|
||||
+ g * scale_stride_group
|
||||
+ pid_token
|
||||
+ qb_indices * scale_stride_k
|
||||
)
|
||||
tl.store(scale_addrs, tl.zeros((CHUNKS_PER_HEAD,), dtype=tl.float32))
|
||||
return
|
||||
|
||||
input_base = o_ptr + pid_token * o_stride_token + global_head * o_stride_head
|
||||
|
||||
HEAD_DIM: tl.constexpr = CHUNKS_PER_HEAD * QUANT_GROUP_SIZE
|
||||
offsets = tl.arange(0, HEAD_DIM)
|
||||
x = tl.load(input_base + offsets).to(tl.float32)
|
||||
|
||||
rope_abs_start: tl.constexpr = (
|
||||
CHUNKS_PER_HEAD - 1
|
||||
) * QUANT_GROUP_SIZE + ROPE_START
|
||||
pos = tl.load(positions_ptr + pid_token)
|
||||
cache_base = cos_sin_cache_ptr + pos * cache_stride_pos
|
||||
is_rope = offsets >= rope_abs_start
|
||||
rope_local = offsets - rope_abs_start
|
||||
|
||||
x_partner = tl.load(input_base + (offsets ^ 1), mask=is_rope, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
cs_idx = tl.maximum(rope_local >> 1, 0)
|
||||
cos_v = tl.load(cache_base + cs_idx, mask=is_rope, other=1.0)
|
||||
sin_v = tl.load(cache_base + HALF_ROPE + cs_idx, mask=is_rope, other=0.0)
|
||||
x_add = x * cos_v + x_partner * sin_v
|
||||
x_sub = x * cos_v - x_partner * sin_v
|
||||
is_even = (rope_local & 1) == 0
|
||||
rotated = tl.where(is_even, x_add, x_sub)
|
||||
x = tl.where(is_rope, rotated, x)
|
||||
|
||||
x_2d = tl.reshape(tl.abs(x), (CHUNKS_PER_HEAD, QUANT_GROUP_SIZE))
|
||||
block_absmax = tl.maximum(tl.max(x_2d, axis=1), eps)
|
||||
scale_raw = block_absmax * (1.0 / fp8_max)
|
||||
scales = tl.math.exp2(tl.ceil(tl.log2(scale_raw)))
|
||||
|
||||
scales_exp = tl.reshape(
|
||||
tl.broadcast_to(
|
||||
tl.reshape(scales, (CHUNKS_PER_HEAD, 1)),
|
||||
(CHUNKS_PER_HEAD, QUANT_GROUP_SIZE),
|
||||
),
|
||||
(HEAD_DIM,),
|
||||
)
|
||||
x_quant = tl.clamp(x / scales_exp, -fp8_max, fp8_max).to(tl.float8e4nv)
|
||||
|
||||
fp8_base = (
|
||||
fp8_ptr
|
||||
+ g * fp8_stride_group
|
||||
+ pid_token * fp8_stride_token
|
||||
+ qb_start * QUANT_GROUP_SIZE
|
||||
)
|
||||
tl.store(fp8_base + offsets, x_quant)
|
||||
|
||||
block_offsets = tl.arange(0, CHUNKS_PER_HEAD)
|
||||
qb_indices = qb_start + block_offsets
|
||||
if TMA_ALIGNED_SCALES:
|
||||
scale_bits = scales.to(tl.int32, bitcast=True)
|
||||
ue8m0_bytes = (scale_bits >> 23) & 0xFF
|
||||
packed_val = tl.sum(ue8m0_bytes << (block_offsets * 8))
|
||||
scale_addr = (
|
||||
scale_ptr
|
||||
+ g * scale_stride_group
|
||||
+ pid_token
|
||||
+ head_in_group * scale_stride_k
|
||||
)
|
||||
tl.store(scale_addr, tl.zeros((), dtype=tl.int32))
|
||||
tl.store(scale_addr, packed_val)
|
||||
else:
|
||||
block_offsets = tl.arange(0, CHUNKS_PER_HEAD)
|
||||
qb_indices = qb_start + block_offsets
|
||||
scale_addrs = (
|
||||
scale_ptr
|
||||
+ g * scale_stride_group
|
||||
+ pid_token
|
||||
+ qb_indices * scale_stride_k
|
||||
)
|
||||
tl.store(scale_addrs, tl.zeros((CHUNKS_PER_HEAD,), dtype=tl.float32))
|
||||
return
|
||||
tl.store(scale_addrs, scales)
|
||||
|
||||
input_base = o_ptr + pid_token * o_stride_token + global_head * o_stride_head
|
||||
|
||||
HEAD_DIM: tl.constexpr = CHUNKS_PER_HEAD * QUANT_GROUP_SIZE
|
||||
offsets = tl.arange(0, HEAD_DIM)
|
||||
x = tl.load(input_base + offsets).to(tl.float32)
|
||||
|
||||
rope_abs_start: tl.constexpr = (CHUNKS_PER_HEAD - 1) * QUANT_GROUP_SIZE + ROPE_START
|
||||
pos = tl.load(positions_ptr + pid_token)
|
||||
cache_base = cos_sin_cache_ptr + pos * cache_stride_pos
|
||||
is_rope = offsets >= rope_abs_start
|
||||
rope_local = offsets - rope_abs_start
|
||||
|
||||
x_partner = tl.load(input_base + (offsets ^ 1), mask=is_rope, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
cs_idx = tl.maximum(rope_local >> 1, 0)
|
||||
cos_v = tl.load(cache_base + cs_idx, mask=is_rope, other=1.0)
|
||||
sin_v = tl.load(cache_base + HALF_ROPE + cs_idx, mask=is_rope, other=0.0)
|
||||
x_add = x * cos_v + x_partner * sin_v
|
||||
x_sub = x * cos_v - x_partner * sin_v
|
||||
is_even = (rope_local & 1) == 0
|
||||
rotated = tl.where(is_even, x_add, x_sub)
|
||||
x = tl.where(is_rope, rotated, x)
|
||||
|
||||
x_2d = tl.reshape(tl.abs(x), (CHUNKS_PER_HEAD, QUANT_GROUP_SIZE))
|
||||
block_absmax = tl.maximum(tl.max(x_2d, axis=1), eps)
|
||||
scale_raw = block_absmax * (1.0 / fp8_max)
|
||||
scales = tl.math.exp2(tl.ceil(tl.log2(scale_raw)))
|
||||
|
||||
scales_exp = tl.reshape(
|
||||
tl.broadcast_to(
|
||||
tl.reshape(scales, (CHUNKS_PER_HEAD, 1)),
|
||||
(CHUNKS_PER_HEAD, QUANT_GROUP_SIZE),
|
||||
),
|
||||
(HEAD_DIM,),
|
||||
)
|
||||
x_quant = tl.clamp(x / scales_exp, -fp8_max, fp8_max).to(tl.float8e4nv)
|
||||
|
||||
fp8_base = (
|
||||
fp8_ptr
|
||||
+ g * fp8_stride_group
|
||||
+ pid_token * fp8_stride_token
|
||||
+ qb_start * QUANT_GROUP_SIZE
|
||||
)
|
||||
tl.store(fp8_base + offsets, x_quant)
|
||||
|
||||
block_offsets = tl.arange(0, CHUNKS_PER_HEAD)
|
||||
qb_indices = qb_start + block_offsets
|
||||
if TMA_ALIGNED_SCALES:
|
||||
scale_bits = scales.to(tl.int32, bitcast=True)
|
||||
ue8m0_bytes = (scale_bits >> 23) & 0xFF
|
||||
packed_val = tl.sum(ue8m0_bytes << (block_offsets * 8))
|
||||
scale_addr = (
|
||||
scale_ptr
|
||||
+ g * scale_stride_group
|
||||
+ pid_token
|
||||
+ head_in_group * scale_stride_k
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
heads_per_group: int,
|
||||
head_dim: int,
|
||||
nope_dim: int,
|
||||
rope_dim: int,
|
||||
quant_group_size: int,
|
||||
runtime_fp8_max: float | None = None,
|
||||
runtime_chunks_per_head: int | None = None,
|
||||
runtime_rope_start: int | None = None,
|
||||
runtime_half_rope: int | None = None,
|
||||
**compile_key_fields: bool,
|
||||
) -> CompileKey:
|
||||
fp8_max = (
|
||||
runtime_fp8_max
|
||||
if runtime_fp8_max is not None
|
||||
else torch.finfo(torch.float8_e4m3fn).max
|
||||
)
|
||||
tl.store(scale_addr, packed_val)
|
||||
else:
|
||||
scale_addrs = (
|
||||
scale_ptr + g * scale_stride_group + pid_token + qb_indices * scale_stride_k
|
||||
chunks_per_head = (
|
||||
runtime_chunks_per_head
|
||||
if runtime_chunks_per_head is not None
|
||||
else head_dim // quant_group_size
|
||||
)
|
||||
rope_start = (
|
||||
runtime_rope_start
|
||||
if runtime_rope_start is not None
|
||||
else nope_dim % quant_group_size
|
||||
)
|
||||
half_rope = (
|
||||
runtime_half_rope if runtime_half_rope is not None else rope_dim // 2
|
||||
)
|
||||
return self.CompileKey(
|
||||
**compile_key_fields,
|
||||
heads_per_group=heads_per_group,
|
||||
fp8_max=fp8_max,
|
||||
quant_group_size=quant_group_size,
|
||||
chunks_per_head=chunks_per_head,
|
||||
rope_start=rope_start,
|
||||
half_rope=half_rope,
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
if hf_config is None:
|
||||
return []
|
||||
|
||||
tp_size = vllm_config.parallel_config.tensor_parallel_size
|
||||
num_heads = int(getattr(hf_config, "num_attention_heads", 0) or 0)
|
||||
num_groups = int(getattr(hf_config, "o_groups", 0) or 0)
|
||||
if num_heads <= 0 or num_groups <= 0:
|
||||
return []
|
||||
|
||||
local_heads = num_heads // tp_size
|
||||
local_groups = num_groups // tp_size
|
||||
if local_groups <= 0:
|
||||
return []
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None:
|
||||
return []
|
||||
|
||||
head_dim = int(getattr(hf_config, "head_dim", 512) or 512)
|
||||
rope_dim = int(getattr(hf_config, "qk_rope_head_dim", 64) or 64)
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
heads_per_group=local_heads // local_groups,
|
||||
head_dim=head_dim,
|
||||
nope_dim=head_dim - rope_dim,
|
||||
rope_dim=rope_dim,
|
||||
quant_group_size=128,
|
||||
tma_aligned_scales=capability.major >= 10,
|
||||
launch_pdl=(False, True),
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
head_dim = compile_key.chunks_per_head * compile_key.quant_group_size
|
||||
fp8_dim = compile_key.heads_per_group * head_dim
|
||||
scale_dtype = torch.int32 if compile_key.tma_aligned_scales else torch.float32
|
||||
return dict(
|
||||
o=TritonWarmupTensor(
|
||||
torch.bfloat16,
|
||||
shape=(1, compile_key.heads_per_group, head_dim),
|
||||
),
|
||||
positions=TritonWarmupTensor(torch.int64),
|
||||
cos_sin_cache=TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, compile_key.half_rope * 2),
|
||||
),
|
||||
fp8_buf=TritonWarmupTensor(
|
||||
torch.float8_e4m3fn,
|
||||
shape=(compile_key.heads_per_group, 1, fp8_dim),
|
||||
strides=(fp8_dim, fp8_dim, 1),
|
||||
),
|
||||
scale_buf=TritonWarmupTensor(
|
||||
scale_dtype,
|
||||
shape=(compile_key.heads_per_group, 1, 16),
|
||||
strides=(16, 16, 16),
|
||||
),
|
||||
num_tokens=1,
|
||||
heads_per_group=compile_key.heads_per_group,
|
||||
quant_group_size=compile_key.quant_group_size,
|
||||
chunks_per_head=compile_key.chunks_per_head,
|
||||
rope_start=compile_key.rope_start,
|
||||
half_rope=compile_key.half_rope,
|
||||
tma_aligned_scales=compile_key.tma_aligned_scales,
|
||||
fp8_max=compile_key.fp8_max,
|
||||
launch_pdl=compile_key.launch_pdl,
|
||||
grid=(1, compile_key.heads_per_group),
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
o: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
fp8_buf: torch.Tensor,
|
||||
scale_buf: torch.Tensor,
|
||||
num_tokens: int,
|
||||
*,
|
||||
heads_per_group: int,
|
||||
quant_group_size: int,
|
||||
chunks_per_head: int,
|
||||
rope_start: int,
|
||||
half_rope: int,
|
||||
tma_aligned_scales: bool,
|
||||
fp8_max: float,
|
||||
launch_pdl: bool,
|
||||
grid: tuple[int, int],
|
||||
) -> LaunchSpec:
|
||||
return grid, dict(
|
||||
fp8_ptr=fp8_buf,
|
||||
scale_ptr=scale_buf,
|
||||
o_stride_token=o.stride(0),
|
||||
o_stride_head=o.stride(1),
|
||||
cache_stride_pos=cos_sin_cache.stride(0),
|
||||
fp8_stride_group=fp8_buf.stride(0),
|
||||
fp8_stride_token=fp8_buf.stride(1),
|
||||
scale_stride_group=scale_buf.stride(0),
|
||||
scale_stride_k=scale_buf.stride(2),
|
||||
eps=1e-10,
|
||||
QUANT_GROUP_SIZE=quant_group_size,
|
||||
CHUNKS_PER_HEAD=chunks_per_head,
|
||||
ROPE_START=rope_start,
|
||||
HALF_ROPE=half_rope,
|
||||
TMA_ALIGNED_SCALES=tma_aligned_scales,
|
||||
launch_pdl=launch_pdl,
|
||||
num_stages=1,
|
||||
num_warps=1,
|
||||
)
|
||||
tl.store(scale_addrs, scales)
|
||||
|
||||
|
||||
def fused_inv_rope_fp8_quant(
|
||||
@@ -256,8 +438,8 @@ def _fused_inv_rope_fp8_quant_kernel_impl(
|
||||
(scale_inner * tma_aligned_T, 1, tma_aligned_T),
|
||||
)
|
||||
grid = (tma_aligned_T, n_groups * heads_per_group)
|
||||
use_gdc = current_platform.is_arch_support_pdl()
|
||||
_fused_inv_rope_fp8_quant_per_head[grid](
|
||||
launch_pdl = current_platform.is_arch_support_pdl()
|
||||
_FUSED_INV_ROPE_FP8_QUANT_KERNEL(
|
||||
o,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
@@ -265,24 +447,14 @@ def _fused_inv_rope_fp8_quant_kernel_impl(
|
||||
scale_buf,
|
||||
num_tokens,
|
||||
heads_per_group=heads_per_group,
|
||||
o_stride_token=o.stride(0),
|
||||
o_stride_head=o.stride(1),
|
||||
cache_stride_pos=cos_sin_cache.stride(0),
|
||||
fp8_stride_group=fp8_buf.stride(0),
|
||||
fp8_stride_token=fp8_buf.stride(1),
|
||||
scale_stride_group=scale_buf.stride(0),
|
||||
scale_stride_k=scale_buf.stride(2),
|
||||
quant_group_size=quant_group_size,
|
||||
chunks_per_head=chunks_per_head,
|
||||
rope_start=rope_start,
|
||||
half_rope=half_rope,
|
||||
tma_aligned_scales=tma_aligned_scales,
|
||||
fp8_max=fp8_max,
|
||||
eps=1e-10,
|
||||
QUANT_GROUP_SIZE=quant_group_size,
|
||||
CHUNKS_PER_HEAD=chunks_per_head,
|
||||
ROPE_START=rope_start,
|
||||
HALF_ROPE=half_rope,
|
||||
TMA_ALIGNED_SCALES=tma_aligned_scales,
|
||||
USE_GDC=use_gdc,
|
||||
launch_pdl=use_gdc,
|
||||
num_stages=1,
|
||||
num_warps=1,
|
||||
launch_pdl=launch_pdl,
|
||||
grid=grid,
|
||||
)
|
||||
return fp8_buf, scale_buf
|
||||
|
||||
@@ -326,3 +498,6 @@ direct_register_custom_op(
|
||||
op_func=_fused_inv_rope_fp8_quant_kernel_impl,
|
||||
fake_impl=_fused_inv_rope_fp8_quant_kernel_fake,
|
||||
)
|
||||
|
||||
|
||||
_FUSED_INV_ROPE_FP8_QUANT_KERNEL = FusedInvRopeFP8QuantKernel()
|
||||
|
||||
@@ -17,9 +17,19 @@ A single grid (T, hc_mult+1) drives both norms: task 0 is enorm on
|
||||
inputs_embeds[token, :], task k+1 is hnorm on previous_hidden_states[token, k, :].
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
kernel_launcher,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -40,164 +50,291 @@ def _rmsnorm_row(
|
||||
tl.store(out_row_ptr + block, y.to(out_row_ptr.dtype.element_ty), mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_mtp_input_rmsnorm_kernel(
|
||||
inputs_embeds_ptr,
|
||||
positions_ptr,
|
||||
prev_hidden_ptr,
|
||||
enorm_weight_ptr,
|
||||
hnorm_weight_ptr,
|
||||
enorm_out_ptr,
|
||||
hnorm_out_ptr,
|
||||
eps,
|
||||
HIDDEN: tl.constexpr,
|
||||
HC_MULT: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
class FusedMTPInputRMSNormKernel(
|
||||
VllmTritonJitKernel["FusedMTPInputRMSNormKernel.CompileKey"]
|
||||
):
|
||||
# int64 token index so per-token offsets don't overflow int32 at
|
||||
# large num_tokens (matches the convention in fused_q_kv_rmsnorm).
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
pid_task = tl.program_id(1)
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
dtype: torch.dtype
|
||||
hidden: int
|
||||
hc_mult: int
|
||||
block_size: int
|
||||
eps: float
|
||||
|
||||
block = tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < HIDDEN
|
||||
@staticmethod
|
||||
@triton.jit
|
||||
def kernel(
|
||||
inputs_embeds_ptr,
|
||||
positions_ptr,
|
||||
prev_hidden_ptr,
|
||||
enorm_weight_ptr,
|
||||
hnorm_weight_ptr,
|
||||
enorm_out_ptr,
|
||||
hnorm_out_ptr,
|
||||
eps,
|
||||
HIDDEN: tl.constexpr,
|
||||
HC_MULT: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
# int64 token index so per-token offsets don't overflow int32 at
|
||||
# large num_tokens (matches the convention in fused_q_kv_rmsnorm).
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
pid_task = tl.program_id(1)
|
||||
|
||||
if pid_task == 0:
|
||||
# enorm path: load inputs_embeds[token, :] then zero-mask at pos==0.
|
||||
# Math is preserved: pos==0 → x=0 → variance=0 → RMSNorm output is 0
|
||||
# regardless of weight, matching torch.where(pos==0, 0, x) + RMSNorm.
|
||||
pos = tl.load(positions_ptr + token_idx)
|
||||
keep = pos != 0
|
||||
x = tl.load(
|
||||
inputs_embeds_ptr + token_idx * HIDDEN + block, mask=mask, other=0.0
|
||||
block = tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < HIDDEN
|
||||
|
||||
if pid_task == 0:
|
||||
# enorm path: load inputs_embeds[token, :] then zero-mask at pos==0.
|
||||
# Math is preserved: pos==0 → x=0 → variance=0 → RMSNorm output is 0
|
||||
# regardless of weight, matching torch.where(pos==0, 0, x) + RMSNorm.
|
||||
pos = tl.load(positions_ptr + token_idx)
|
||||
keep = pos != 0
|
||||
x = tl.load(
|
||||
inputs_embeds_ptr + token_idx * HIDDEN + block, mask=mask, other=0.0
|
||||
)
|
||||
x = tl.where(keep, x, 0.0)
|
||||
_rmsnorm_row(
|
||||
x,
|
||||
enorm_weight_ptr,
|
||||
enorm_out_ptr + token_idx * HIDDEN,
|
||||
block,
|
||||
mask,
|
||||
eps,
|
||||
HIDDEN,
|
||||
)
|
||||
else:
|
||||
# hnorm path: load prev_hidden[token, slot, :].
|
||||
slot = pid_task - 1
|
||||
row_offset = (token_idx * HC_MULT + slot) * HIDDEN
|
||||
x = tl.load(prev_hidden_ptr + row_offset + block, mask=mask, other=0.0)
|
||||
_rmsnorm_row(
|
||||
x,
|
||||
hnorm_weight_ptr,
|
||||
hnorm_out_ptr + row_offset,
|
||||
block,
|
||||
mask,
|
||||
eps,
|
||||
HIDDEN,
|
||||
)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
hidden: int,
|
||||
hc_mult: int,
|
||||
eps: float,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
dtype=dtype,
|
||||
hidden=hidden,
|
||||
hc_mult=hc_mult,
|
||||
block_size=next_power_of_2(hidden),
|
||||
eps=eps,
|
||||
)
|
||||
x = tl.where(keep, x, 0.0)
|
||||
_rmsnorm_row(
|
||||
x,
|
||||
enorm_weight_ptr,
|
||||
enorm_out_ptr + token_idx * HIDDEN,
|
||||
block,
|
||||
mask,
|
||||
eps,
|
||||
HIDDEN,
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
if int(getattr(hf_config, "num_nextn_predict_layers", 0) or 0) <= 0:
|
||||
return []
|
||||
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
hidden=int(hf_config.hidden_size),
|
||||
hc_mult=int(hf_config.hc_mult),
|
||||
eps=float(hf_config.rms_norm_eps),
|
||||
)
|
||||
else:
|
||||
# hnorm path: load prev_hidden[token, slot, :].
|
||||
slot = pid_task - 1
|
||||
row_offset = (token_idx * HC_MULT + slot) * HIDDEN
|
||||
x = tl.load(prev_hidden_ptr + row_offset + block, mask=mask, other=0.0)
|
||||
_rmsnorm_row(
|
||||
x,
|
||||
hnorm_weight_ptr,
|
||||
hnorm_out_ptr + row_offset,
|
||||
block,
|
||||
mask,
|
||||
eps,
|
||||
HIDDEN,
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
return dict(
|
||||
inputs_embeds=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.hidden),
|
||||
),
|
||||
positions=TritonWarmupTensor(torch.int64),
|
||||
previous_hidden_states=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.hc_mult, compile_key.hidden),
|
||||
),
|
||||
enorm_weight=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(compile_key.hidden,),
|
||||
),
|
||||
hnorm_weight=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(compile_key.hidden,),
|
||||
),
|
||||
_outputs=(
|
||||
TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.hidden),
|
||||
),
|
||||
TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.hc_mult, compile_key.hidden),
|
||||
),
|
||||
),
|
||||
eps=compile_key.eps,
|
||||
hc_mult=compile_key.hc_mult,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
inputs_embeds: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
enorm_weight: torch.Tensor,
|
||||
hnorm_weight: torch.Tensor,
|
||||
eps: float,
|
||||
hc_mult: int,
|
||||
*,
|
||||
_outputs: tuple[Any, Any] | None = None,
|
||||
) -> LaunchSpec:
|
||||
num_tokens, hidden = inputs_embeds.shape
|
||||
if _outputs is None:
|
||||
assert inputs_embeds.ndim == 2
|
||||
assert previous_hidden_states.ndim == 3
|
||||
assert previous_hidden_states.shape[1] == hc_mult
|
||||
assert inputs_embeds.shape[0] == previous_hidden_states.shape[0], (
|
||||
"token dim mismatch"
|
||||
)
|
||||
assert (
|
||||
hidden
|
||||
== previous_hidden_states.shape[2]
|
||||
== enorm_weight.shape[0]
|
||||
== hnorm_weight.shape[0]
|
||||
)
|
||||
assert (
|
||||
inputs_embeds.is_contiguous() and previous_hidden_states.is_contiguous()
|
||||
)
|
||||
assert enorm_weight.is_contiguous() and hnorm_weight.is_contiguous()
|
||||
_outputs = (
|
||||
torch.empty_like(inputs_embeds),
|
||||
torch.empty_like(previous_hidden_states),
|
||||
)
|
||||
|
||||
enorm_out, hnorm_out = _outputs
|
||||
grid = None if num_tokens == 0 else (num_tokens, hc_mult + 1)
|
||||
return (
|
||||
grid,
|
||||
dict(
|
||||
prev_hidden_ptr=previous_hidden_states,
|
||||
enorm_out_ptr=enorm_out,
|
||||
hnorm_out_ptr=hnorm_out,
|
||||
HIDDEN=hidden,
|
||||
HC_MULT=hc_mult,
|
||||
BLOCK_SIZE=next_power_of_2(hidden),
|
||||
),
|
||||
_outputs,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _mtp_shared_head_rmsnorm_kernel(
|
||||
x_ptr,
|
||||
weight_ptr,
|
||||
out_ptr,
|
||||
eps,
|
||||
HIDDEN: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
class MTPSharedHeadRMSNormKernel(
|
||||
VllmTritonJitKernel["MTPSharedHeadRMSNormKernel.CompileKey"]
|
||||
):
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
block = tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < HIDDEN
|
||||
x = tl.load(x_ptr + token_idx * HIDDEN + block, mask=mask, other=0.0)
|
||||
_rmsnorm_row(
|
||||
x,
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
dtype: torch.dtype
|
||||
hidden: int
|
||||
block_size: int
|
||||
eps: float
|
||||
|
||||
@staticmethod
|
||||
@triton.jit
|
||||
def kernel(
|
||||
x_ptr,
|
||||
weight_ptr,
|
||||
out_ptr + token_idx * HIDDEN,
|
||||
block,
|
||||
mask,
|
||||
out_ptr,
|
||||
eps,
|
||||
HIDDEN,
|
||||
)
|
||||
HIDDEN: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
block = tl.arange(0, BLOCK_SIZE)
|
||||
mask = block < HIDDEN
|
||||
x = tl.load(x_ptr + token_idx * HIDDEN + block, mask=mask, other=0.0)
|
||||
_rmsnorm_row(
|
||||
x,
|
||||
weight_ptr,
|
||||
out_ptr + token_idx * HIDDEN,
|
||||
block,
|
||||
mask,
|
||||
eps,
|
||||
HIDDEN,
|
||||
)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
hidden: int,
|
||||
eps: float,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
dtype=dtype,
|
||||
hidden=hidden,
|
||||
block_size=next_power_of_2(hidden),
|
||||
eps=eps,
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
if int(getattr(hf_config, "num_nextn_predict_layers", 0) or 0) <= 0:
|
||||
return []
|
||||
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
hidden=int(hf_config.hidden_size),
|
||||
eps=float(hf_config.rms_norm_eps),
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
return dict(
|
||||
hidden_states=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.hidden),
|
||||
),
|
||||
weight=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(compile_key.hidden,),
|
||||
),
|
||||
_output=TritonWarmupTensor(
|
||||
compile_key.dtype,
|
||||
shape=(1, compile_key.hidden),
|
||||
),
|
||||
eps=compile_key.eps,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
eps: float,
|
||||
*,
|
||||
_output: Any | None = None,
|
||||
) -> LaunchSpec:
|
||||
num_tokens, hidden = hidden_states.shape
|
||||
if _output is None:
|
||||
assert hidden_states.ndim == 2
|
||||
assert hidden_states.is_contiguous()
|
||||
assert weight.is_contiguous()
|
||||
_output = torch.empty_like(hidden_states)
|
||||
|
||||
grid = None if num_tokens == 0 else (num_tokens,)
|
||||
return (
|
||||
grid,
|
||||
dict(
|
||||
x_ptr=hidden_states,
|
||||
out_ptr=_output,
|
||||
HIDDEN=hidden,
|
||||
BLOCK_SIZE=next_power_of_2(hidden),
|
||||
),
|
||||
_output,
|
||||
)
|
||||
|
||||
|
||||
def mtp_shared_head_rmsnorm(
|
||||
hidden_states: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
eps: float,
|
||||
) -> torch.Tensor:
|
||||
"""RMSNorm for MTP's SharedHead.norm, on (T, H) bf16 input.
|
||||
|
||||
Uses the same ``_rmsnorm_row`` body as ``fused_mtp_input_rmsnorm`` so the
|
||||
MTP draft path runs one consistent RMSNorm implementation end to end.
|
||||
"""
|
||||
assert hidden_states.ndim == 2
|
||||
assert hidden_states.is_contiguous()
|
||||
assert weight.is_contiguous()
|
||||
num_tokens, hidden = hidden_states.shape
|
||||
out = torch.empty_like(hidden_states)
|
||||
if num_tokens == 0:
|
||||
return out
|
||||
block_size = triton.next_power_of_2(hidden)
|
||||
_mtp_shared_head_rmsnorm_kernel[(num_tokens,)](
|
||||
hidden_states,
|
||||
weight,
|
||||
out,
|
||||
eps,
|
||||
HIDDEN=hidden,
|
||||
BLOCK_SIZE=block_size,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def fused_mtp_input_rmsnorm(
|
||||
inputs_embeds: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
enorm_weight: torch.Tensor,
|
||||
hnorm_weight: torch.Tensor,
|
||||
eps: float,
|
||||
hc_mult: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Returns (enorm_out, hnorm_out).
|
||||
|
||||
enorm_out has the same shape as inputs_embeds (2D, [T, H]).
|
||||
hnorm_out has the same shape as previous_hidden_states (3D, [T, hc_mult, H]).
|
||||
previous_hidden_states must already be reshaped to 3D.
|
||||
"""
|
||||
assert inputs_embeds.ndim == 2
|
||||
assert previous_hidden_states.ndim == 3
|
||||
assert previous_hidden_states.shape[1] == hc_mult
|
||||
assert inputs_embeds.shape[0] == previous_hidden_states.shape[0], (
|
||||
"token dim mismatch"
|
||||
)
|
||||
assert (
|
||||
inputs_embeds.shape[1]
|
||||
== previous_hidden_states.shape[2]
|
||||
== enorm_weight.shape[0]
|
||||
== hnorm_weight.shape[0]
|
||||
)
|
||||
assert inputs_embeds.is_contiguous() and previous_hidden_states.is_contiguous()
|
||||
assert enorm_weight.is_contiguous() and hnorm_weight.is_contiguous()
|
||||
|
||||
num_tokens, hidden = inputs_embeds.shape
|
||||
enorm_out = torch.empty_like(inputs_embeds)
|
||||
hnorm_out = torch.empty_like(previous_hidden_states)
|
||||
if num_tokens == 0:
|
||||
return enorm_out, hnorm_out
|
||||
|
||||
block_size = triton.next_power_of_2(hidden)
|
||||
_fused_mtp_input_rmsnorm_kernel[(num_tokens, hc_mult + 1)](
|
||||
inputs_embeds,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
enorm_weight,
|
||||
hnorm_weight,
|
||||
enorm_out,
|
||||
hnorm_out,
|
||||
eps,
|
||||
HIDDEN=hidden,
|
||||
HC_MULT=hc_mult,
|
||||
BLOCK_SIZE=block_size,
|
||||
)
|
||||
return enorm_out, hnorm_out
|
||||
_FUSED_MTP_INPUT_RMSNORM_KERNEL = FusedMTPInputRMSNormKernel()
|
||||
_MTP_SHARED_HEAD_RMSNORM_KERNEL = MTPSharedHeadRMSNormKernel()
|
||||
|
||||
@@ -1,101 +1,193 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
kernel_launcher,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
|
||||
|
||||
def save_partial_states(
|
||||
kv: torch.Tensor,
|
||||
score: torch.Tensor,
|
||||
ape: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
state_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
block_size: int,
|
||||
state_width: int,
|
||||
compress_ratio: int,
|
||||
pdl_kwargs: dict | None = None,
|
||||
) -> None:
|
||||
"""Write packed [kv, score+ape] partial states into the compressor cache.
|
||||
|
||||
One program per token; pads (slot_id == -1) are skipped.
|
||||
"""
|
||||
num_actual = slot_mapping.shape[0]
|
||||
head_size = kv.shape[-1]
|
||||
_save_partial_states_kernel[(num_actual,)](
|
||||
kv,
|
||||
kv.stride(0),
|
||||
score,
|
||||
score.stride(0),
|
||||
ape,
|
||||
ape.stride(0),
|
||||
positions,
|
||||
state_cache,
|
||||
state_cache.stride(0),
|
||||
state_cache.stride(1),
|
||||
slot_mapping,
|
||||
block_size,
|
||||
HEAD_SIZE=head_size,
|
||||
TRITON_BLOCK_SIZE=triton.next_power_of_2(head_size),
|
||||
STATE_WIDTH=state_width,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
**(pdl_kwargs or {}),
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _save_partial_states_kernel(
|
||||
kv_ptr,
|
||||
kv_stride,
|
||||
score_ptr,
|
||||
score_stride,
|
||||
ape_ptr,
|
||||
ape_stride,
|
||||
positions_ptr,
|
||||
state_cache_ptr,
|
||||
state_cache_stride0,
|
||||
state_cache_stride1,
|
||||
slot_mapping_ptr,
|
||||
block_size,
|
||||
HEAD_SIZE: tl.constexpr,
|
||||
TRITON_BLOCK_SIZE: tl.constexpr,
|
||||
# state_cache last dim packs [kv_state, score_state], each STATE_WIDTH wide.
|
||||
STATE_WIDTH: tl.constexpr,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
class SavePartialStatesKernel(
|
||||
VllmTritonJitKernel["SavePartialStatesKernel.CompileKey"]
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
slot_id = tl.load(slot_mapping_ptr + token_idx)
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
head_size: int
|
||||
triton_block_size: int
|
||||
state_width: int
|
||||
compress_ratio: int
|
||||
kv_stride: int
|
||||
score_stride: int
|
||||
ape_stride: int
|
||||
state_cache_stride0: int
|
||||
state_cache_stride1: int
|
||||
block_size: int
|
||||
|
||||
# Skip padded / invalid tokens (slot_id == -1 is the PAD sentinel used
|
||||
# by vLLM). During CUDA graph replay the batch may contain padding
|
||||
# tokens whose slot_mapping is -1; writing to kv_state[-1] would be an
|
||||
# illegal memory access.
|
||||
if slot_id < 0:
|
||||
return
|
||||
@staticmethod
|
||||
@triton.jit
|
||||
def kernel(
|
||||
kv_ptr,
|
||||
kv_stride,
|
||||
score_ptr,
|
||||
score_stride,
|
||||
ape_ptr,
|
||||
ape_stride,
|
||||
positions_ptr,
|
||||
state_cache_ptr,
|
||||
state_cache_stride0,
|
||||
state_cache_stride1,
|
||||
slot_mapping_ptr,
|
||||
block_size,
|
||||
HEAD_SIZE: tl.constexpr,
|
||||
TRITON_BLOCK_SIZE: tl.constexpr,
|
||||
# state_cache last dim packs [kv_state, score_state], each STATE_WIDTH wide.
|
||||
STATE_WIDTH: tl.constexpr,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
slot_id = tl.load(slot_mapping_ptr + token_idx)
|
||||
|
||||
block_idx = slot_id // block_size
|
||||
pos_in_block = slot_id % block_size
|
||||
base_ptr = (
|
||||
state_cache_ptr
|
||||
+ block_idx * state_cache_stride0
|
||||
+ pos_in_block * state_cache_stride1
|
||||
)
|
||||
# Skip padded / invalid tokens (slot_id == -1 is the PAD sentinel used
|
||||
# by vLLM). During CUDA graph replay the batch may contain padding
|
||||
# tokens whose slot_mapping is -1; writing to kv_state[-1] would be an
|
||||
# illegal memory access.
|
||||
if slot_id < 0:
|
||||
return
|
||||
|
||||
block = tl.arange(0, TRITON_BLOCK_SIZE)
|
||||
mask = block < HEAD_SIZE
|
||||
block_idx = slot_id // block_size
|
||||
pos_in_block = slot_id % block_size
|
||||
base_ptr = (
|
||||
state_cache_ptr
|
||||
+ block_idx * state_cache_stride0
|
||||
+ pos_in_block * state_cache_stride1
|
||||
)
|
||||
|
||||
kv = tl.load(kv_ptr + token_idx * kv_stride + block, mask=mask)
|
||||
tl.store(base_ptr + block, kv, mask=mask)
|
||||
block = tl.arange(0, TRITON_BLOCK_SIZE)
|
||||
mask = block < HEAD_SIZE
|
||||
|
||||
# Fused: score += ape[position % compress_ratio]
|
||||
position = tl.load(positions_ptr + token_idx)
|
||||
ape_row = position % COMPRESS_RATIO
|
||||
ape = tl.load(ape_ptr + ape_row * ape_stride + block, mask=mask)
|
||||
score = tl.load(score_ptr + token_idx * score_stride + block, mask=mask)
|
||||
tl.store(
|
||||
base_ptr + STATE_WIDTH + block,
|
||||
score + ape,
|
||||
mask=mask,
|
||||
)
|
||||
kv = tl.load(kv_ptr + token_idx * kv_stride + block, mask=mask)
|
||||
tl.store(base_ptr + block, kv, mask=mask)
|
||||
|
||||
# Fused: score += ape[position % compress_ratio]
|
||||
position = tl.load(positions_ptr + token_idx)
|
||||
ape_row = position % COMPRESS_RATIO
|
||||
ape = tl.load(ape_ptr + ape_row * ape_stride + block, mask=mask)
|
||||
score = tl.load(score_ptr + token_idx * score_stride + block, mask=mask)
|
||||
tl.store(
|
||||
base_ptr + STATE_WIDTH + block,
|
||||
score + ape,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
head_size: int,
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
**compile_key_fields,
|
||||
head_size=head_size,
|
||||
triton_block_size=next_power_of_2(head_size),
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
*,
|
||||
head_dim: int,
|
||||
compress_ratio: int,
|
||||
) -> list[CompileKey]:
|
||||
if head_dim <= 0 or compress_ratio not in (4, 128):
|
||||
return []
|
||||
|
||||
coefficient = 2 if compress_ratio == 4 else 1
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
head_size=coefficient * head_dim,
|
||||
state_width=coefficient * head_dim,
|
||||
compress_ratio=compress_ratio,
|
||||
kv_stride=2 * coefficient * head_dim,
|
||||
score_stride=2 * coefficient * head_dim,
|
||||
ape_stride=coefficient * head_dim,
|
||||
state_cache_stride0=8 * coefficient * head_dim,
|
||||
state_cache_stride1=2 * coefficient * head_dim,
|
||||
block_size=4 if compress_ratio == 4 else 8,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
return dict(
|
||||
kv=TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, compile_key.head_size),
|
||||
strides=(compile_key.kv_stride, 1),
|
||||
),
|
||||
score=TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, 1),
|
||||
strides=(compile_key.score_stride, 1),
|
||||
),
|
||||
ape=TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, 1),
|
||||
strides=(compile_key.ape_stride, 1),
|
||||
),
|
||||
positions=TritonWarmupTensor(torch.int64),
|
||||
state_cache=TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(1, 1, compile_key.state_width),
|
||||
strides=(
|
||||
compile_key.state_cache_stride0,
|
||||
compile_key.state_cache_stride1,
|
||||
1,
|
||||
),
|
||||
),
|
||||
slot_mapping=TritonWarmupTensor(torch.int64),
|
||||
block_size=compile_key.block_size,
|
||||
state_width=compile_key.state_width,
|
||||
compress_ratio=compile_key.compress_ratio,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
kv: torch.Tensor,
|
||||
score: torch.Tensor,
|
||||
ape: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
state_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
block_size: int,
|
||||
state_width: int,
|
||||
compress_ratio: int,
|
||||
pdl_kwargs: dict | None = None,
|
||||
) -> LaunchSpec:
|
||||
"""Write packed [kv, score+ape] partial states into the compressor cache.
|
||||
|
||||
One program per token; pads (slot_id == -1) are skipped.
|
||||
"""
|
||||
num_actual = slot_mapping.shape[0]
|
||||
head_size = kv.shape[-1]
|
||||
return (num_actual,), dict(
|
||||
kv_stride=kv.stride(0),
|
||||
score_stride=score.stride(0),
|
||||
ape_stride=ape.stride(0),
|
||||
state_cache_stride0=state_cache.stride(0),
|
||||
state_cache_stride1=state_cache.stride(1),
|
||||
block_size=block_size,
|
||||
HEAD_SIZE=head_size,
|
||||
TRITON_BLOCK_SIZE=next_power_of_2(head_size),
|
||||
STATE_WIDTH=state_width,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
**(pdl_kwargs or {}),
|
||||
)
|
||||
|
||||
|
||||
_SAVE_PARTIAL_STATES_KERNEL = SavePartialStatesKernel()
|
||||
|
||||
@@ -18,7 +18,7 @@ from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import (
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.ops.fused_indexer_q import MXFP4_BLOCK_SIZE
|
||||
from vllm.models.deepseek_v4.common.ops.save_partial_states import (
|
||||
save_partial_states,
|
||||
_SAVE_PARTIAL_STATES_KERNEL,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backend import (
|
||||
@@ -306,6 +306,18 @@ class DeepseekCompressor(nn.Module):
|
||||
f"Unsupported head_dim for fused quant+cache: {self.head_dim}"
|
||||
)
|
||||
|
||||
if vllm_config.kernel_config.enable_jit_warmup:
|
||||
_SAVE_PARTIAL_STATES_KERNEL.register_warmup(
|
||||
head_dim=self.head_dim,
|
||||
compress_ratio=self.compress_ratio,
|
||||
)
|
||||
if self.head_dim != 512:
|
||||
from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import ( # noqa: E501
|
||||
_FUSED_KV_COMPRESS_NORM_ROPE_INSERT_INDEXER_TRITON_KERNEL,
|
||||
)
|
||||
|
||||
_FUSED_KV_COMPRESS_NORM_ROPE_INSERT_INDEXER_TRITON_KERNEL.register_warmup()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
# [num_tokens, 2 * self.coff * self.head_dim]
|
||||
@@ -351,7 +363,7 @@ class DeepseekCompressor(nn.Module):
|
||||
# GEMM; state_cache from this kernel) but neither emits/waits on PDL
|
||||
# grid dependency primitives, so launch_pdl=True caused a
|
||||
# read-after-write race and non-deterministic output.
|
||||
save_partial_states(
|
||||
_SAVE_PARTIAL_STATES_KERNEL(
|
||||
kv=kv,
|
||||
score=score,
|
||||
ape=self.ape,
|
||||
|
||||
@@ -51,9 +51,9 @@ from vllm.models.common.ops.sequence_parallel import (
|
||||
sp_padding_mask,
|
||||
sp_shard,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.ops import (
|
||||
fused_mtp_input_rmsnorm,
|
||||
mtp_shared_head_rmsnorm,
|
||||
from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import (
|
||||
_FUSED_MTP_INPUT_RMSNORM_KERNEL,
|
||||
_MTP_SHARED_HEAD_RMSNORM_KERNEL,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
@@ -141,6 +141,10 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
aux_stream_list=aux_stream_list,
|
||||
)
|
||||
|
||||
if vllm_config.kernel_config.enable_jit_warmup:
|
||||
_FUSED_MTP_INPUT_RMSNORM_KERNEL.register_warmup()
|
||||
_MTP_SHARED_HEAD_RMSNORM_KERNEL.register_warmup()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
@@ -157,7 +161,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
-1, self.hc_mult, self.config.hidden_size
|
||||
)
|
||||
# Fused: mask inputs at position 0 (not needed by MTP), enorm, hnorm.
|
||||
inputs_embeds, previous_hidden_states = fused_mtp_input_rmsnorm(
|
||||
inputs_embeds, previous_hidden_states = _FUSED_MTP_INPUT_RMSNORM_KERNEL(
|
||||
inputs_embeds,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
@@ -270,7 +274,7 @@ class DeepSeekV4MultiTokenPredictor(nn.Module):
|
||||
mtp_layer.rms_norm_eps,
|
||||
mtp_layer.hc_eps,
|
||||
)
|
||||
hidden_states = mtp_shared_head_rmsnorm(
|
||||
hidden_states = _MTP_SHARED_HEAD_RMSNORM_KERNEL(
|
||||
hidden_states,
|
||||
mtp_layer.shared_head.norm.weight.data,
|
||||
mtp_layer.shared_head.norm.variance_epsilon,
|
||||
|
||||
@@ -9,6 +9,12 @@ import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
kernel_launcher,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.triton_utils import tl, triton
|
||||
@@ -338,90 +344,180 @@ def build_c128a_topk_metadata(
|
||||
if num_tokens == 0:
|
||||
return global_decode, decode_lens, prefill_local
|
||||
|
||||
_build_c128a_topk_metadata_kernel[(num_tokens,)](
|
||||
_BUILD_C128A_TOPK_METADATA_KERNEL(
|
||||
global_decode_buffer,
|
||||
global_decode_buffer.stride(0),
|
||||
decode_lens_buffer,
|
||||
prefill_buffer,
|
||||
prefill_buffer.stride(0),
|
||||
positions,
|
||||
compress_ratio,
|
||||
max_compressed_tokens,
|
||||
num_decode_tokens,
|
||||
token_to_req_indices,
|
||||
block_table,
|
||||
block_table.stride(0),
|
||||
block_size,
|
||||
slot_mapping,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
return global_decode, decode_lens, prefill_local
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _build_c128a_topk_metadata_kernel(
|
||||
# Decode outputs
|
||||
global_decode_ptr,
|
||||
global_decode_stride,
|
||||
decode_lens_ptr,
|
||||
# Prefill output
|
||||
prefill_local_ptr,
|
||||
prefill_local_stride,
|
||||
# Inputs
|
||||
positions_ptr,
|
||||
compress_ratio,
|
||||
max_compressed_tokens,
|
||||
num_decode_tokens,
|
||||
token_to_req_indices_ptr,
|
||||
block_table_ptr,
|
||||
block_table_stride,
|
||||
block_size,
|
||||
slot_mapping_ptr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
class BuildC128ATopkMetadataKernel(
|
||||
VllmTritonJitKernel["BuildC128ATopkMetadataKernel.CompileKey"]
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
position = tl.load(positions_ptr + token_idx)
|
||||
num_compressed = (position + 1) // compress_ratio
|
||||
num_compressed = tl.minimum(num_compressed, max_compressed_tokens)
|
||||
is_decode = token_idx < num_decode_tokens
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
compress_ratio: int
|
||||
max_compressed_tokens: int
|
||||
block_size: int
|
||||
triton_block_size: int
|
||||
|
||||
if is_decode:
|
||||
# --- Decode: block-table lookup → global slot ids + count ---
|
||||
is_valid_token = tl.load(slot_mapping_ptr + token_idx) >= 0
|
||||
req_idx = tl.load(token_to_req_indices_ptr + token_idx)
|
||||
count = tl.zeros((), dtype=tl.int32)
|
||||
for i in range(0, max_compressed_tokens, BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offset < max_compressed_tokens
|
||||
is_valid = offset < num_compressed
|
||||
@staticmethod
|
||||
@triton.jit(
|
||||
do_not_specialize=[
|
||||
"global_decode_stride",
|
||||
"prefill_local_stride",
|
||||
"num_decode_tokens",
|
||||
"block_table_stride",
|
||||
]
|
||||
)
|
||||
def kernel(
|
||||
# Decode outputs
|
||||
global_decode_ptr,
|
||||
global_decode_stride,
|
||||
decode_lens_ptr,
|
||||
# Prefill output
|
||||
prefill_local_ptr,
|
||||
prefill_local_stride,
|
||||
# Inputs
|
||||
positions_ptr,
|
||||
compress_ratio,
|
||||
max_compressed_tokens,
|
||||
num_decode_tokens,
|
||||
token_to_req_indices_ptr,
|
||||
block_table_ptr,
|
||||
block_table_stride,
|
||||
block_size,
|
||||
slot_mapping_ptr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
token_idx = tl.program_id(0)
|
||||
position = tl.load(positions_ptr + token_idx)
|
||||
num_compressed = (position + 1) // compress_ratio
|
||||
num_compressed = tl.minimum(num_compressed, max_compressed_tokens)
|
||||
is_decode = token_idx < num_decode_tokens
|
||||
|
||||
if is_decode:
|
||||
# --- Decode: block-table lookup → global slot ids + count ---
|
||||
is_valid_token = tl.load(slot_mapping_ptr + token_idx) >= 0
|
||||
req_idx = tl.load(token_to_req_indices_ptr + token_idx)
|
||||
count = tl.zeros((), dtype=tl.int32)
|
||||
for i in range(0, max_compressed_tokens, BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offset < max_compressed_tokens
|
||||
is_valid = offset < num_compressed
|
||||
|
||||
block_indices = offset // block_size
|
||||
block_numbers = tl.load(
|
||||
block_table_ptr + req_idx * block_table_stride + block_indices,
|
||||
mask=mask & is_valid,
|
||||
)
|
||||
block_offsets = offset % block_size
|
||||
slot_ids = block_numbers * block_size + block_offsets
|
||||
slot_ids = tl.where(is_valid, slot_ids, -1)
|
||||
tl.store(
|
||||
global_decode_ptr + token_idx * global_decode_stride + offset,
|
||||
slot_ids,
|
||||
mask=mask,
|
||||
)
|
||||
count += tl.sum(is_valid.to(tl.int32), axis=0)
|
||||
|
||||
block_indices = offset // block_size
|
||||
block_numbers = tl.load(
|
||||
block_table_ptr + req_idx * block_table_stride + block_indices,
|
||||
mask=mask & is_valid,
|
||||
)
|
||||
block_offsets = offset % block_size
|
||||
slot_ids = block_numbers * block_size + block_offsets
|
||||
slot_ids = tl.where(is_valid, slot_ids, -1)
|
||||
tl.store(
|
||||
global_decode_ptr + token_idx * global_decode_stride + offset,
|
||||
slot_ids,
|
||||
mask=mask,
|
||||
decode_lens_ptr + token_idx,
|
||||
tl.where(is_valid_token, count, 0),
|
||||
)
|
||||
count += tl.sum(is_valid.to(tl.int32), axis=0)
|
||||
else:
|
||||
# --- Prefill: write local indices ---
|
||||
pfx_idx = token_idx - num_decode_tokens
|
||||
for i in range(0, max_compressed_tokens, BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offset < max_compressed_tokens
|
||||
tl.store(
|
||||
prefill_local_ptr + pfx_idx * prefill_local_stride + offset,
|
||||
tl.where(offset < num_compressed, offset, -1),
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
tl.store(
|
||||
decode_lens_ptr + token_idx,
|
||||
tl.where(is_valid_token, count, 0),
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(**compile_key_fields)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
if getattr(hf_config, "model_type", None) != "deepseek_v4":
|
||||
return []
|
||||
|
||||
compress_ratio = 128
|
||||
max_model_len = vllm_config.model_config.max_model_len
|
||||
if max_model_len <= 0:
|
||||
return []
|
||||
max_compressed_tokens = cdiv(max_model_len, compress_ratio)
|
||||
max_compressed_tokens = (
|
||||
cdiv(max_compressed_tokens, _C128A_TOPK_ALIGNMENT) * _C128A_TOPK_ALIGNMENT
|
||||
)
|
||||
else:
|
||||
# --- Prefill: write local indices ---
|
||||
pfx_idx = token_idx - num_decode_tokens
|
||||
for i in range(0, max_compressed_tokens, BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offset < max_compressed_tokens
|
||||
tl.store(
|
||||
prefill_local_ptr + pfx_idx * prefill_local_stride + offset,
|
||||
tl.where(offset < num_compressed, offset, -1),
|
||||
mask=mask,
|
||||
)
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
compress_ratio=compress_ratio,
|
||||
max_compressed_tokens=max_compressed_tokens,
|
||||
# DeepSeek V4 sparse MLA uses 256-token KV pages; C128A metadata
|
||||
# works in compressed-token units.
|
||||
block_size=256 // compress_ratio,
|
||||
triton_block_size=1024,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
int32_ptr = TritonWarmupTensor(torch.int32)
|
||||
return dict(
|
||||
global_decode_buffer=TritonWarmupTensor(
|
||||
torch.int32, shape=(1, 1), strides=(1, 1)
|
||||
),
|
||||
decode_lens_buffer=int32_ptr,
|
||||
prefill_buffer=TritonWarmupTensor(
|
||||
torch.int32, shape=(1, 1), strides=(1, 1)
|
||||
),
|
||||
positions=TritonWarmupTensor(torch.int64),
|
||||
compress_ratio=compile_key.compress_ratio,
|
||||
max_compressed_tokens=compile_key.max_compressed_tokens,
|
||||
num_decode_tokens=0,
|
||||
token_to_req_indices=int32_ptr,
|
||||
block_table=TritonWarmupTensor(torch.int32, shape=(1, 1), strides=(1, 1)),
|
||||
block_size=compile_key.block_size,
|
||||
slot_mapping=TritonWarmupTensor(torch.int64),
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
global_decode_buffer: torch.Tensor,
|
||||
decode_lens_buffer: torch.Tensor,
|
||||
prefill_buffer: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
compress_ratio: int,
|
||||
max_compressed_tokens: int,
|
||||
num_decode_tokens: int,
|
||||
token_to_req_indices: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
block_size: int,
|
||||
slot_mapping: torch.Tensor,
|
||||
) -> LaunchSpec:
|
||||
return (positions.shape[0],), dict(
|
||||
global_decode_ptr=global_decode_buffer,
|
||||
global_decode_stride=global_decode_buffer.stride(0),
|
||||
decode_lens_ptr=decode_lens_buffer,
|
||||
prefill_local_ptr=prefill_buffer,
|
||||
prefill_local_stride=prefill_buffer.stride(0),
|
||||
block_table_stride=block_table.stride(0),
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
|
||||
|
||||
_BUILD_C128A_TOPK_METADATA_KERNEL = BuildC128ATopkMetadataKernel()
|
||||
|
||||
@@ -39,9 +39,9 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.deepseek_mtp import SharedHead
|
||||
from vllm.model_executor.models.deepseek_v2 import get_spec_layer_idx_from_weight_name
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.models.deepseek_v4.common.ops import (
|
||||
fused_mtp_input_rmsnorm,
|
||||
mtp_shared_head_rmsnorm,
|
||||
from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import (
|
||||
_FUSED_MTP_INPUT_RMSNORM_KERNEL,
|
||||
_MTP_SHARED_HEAD_RMSNORM_KERNEL,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -130,6 +130,10 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
|
||||
self.hc_head_op = HCHeadOp()
|
||||
|
||||
if vllm_config.kernel_config.enable_jit_warmup:
|
||||
_FUSED_MTP_INPUT_RMSNORM_KERNEL.register_warmup()
|
||||
_MTP_SHARED_HEAD_RMSNORM_KERNEL.register_warmup()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
@@ -146,7 +150,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
-1, self.hc_mult, self.config.hidden_size
|
||||
)
|
||||
# Fused: mask inputs at position 0 (not needed by MTP), enorm, hnorm.
|
||||
inputs_embeds, previous_hidden_states = fused_mtp_input_rmsnorm(
|
||||
inputs_embeds, previous_hidden_states = _FUSED_MTP_INPUT_RMSNORM_KERNEL(
|
||||
inputs_embeds,
|
||||
positions,
|
||||
previous_hidden_states,
|
||||
@@ -253,7 +257,7 @@ class DeepSeekV4MultiTokenPredictor(nn.Module):
|
||||
mtp_layer.rms_norm_eps,
|
||||
mtp_layer.hc_eps,
|
||||
)
|
||||
hidden_states = mtp_shared_head_rmsnorm(
|
||||
hidden_states = _MTP_SHARED_HEAD_RMSNORM_KERNEL(
|
||||
hidden_states,
|
||||
mtp_layer.shared_head.norm.weight.data,
|
||||
mtp_layer.shared_head.norm.variance_epsilon,
|
||||
|
||||
Reference in New Issue
Block a user