[5/N][warmup][DSv4] Migrate NVIDIA CuTeDSL attention kernels (#53566)

Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Roberto L. Castro
2026-09-12 00:05:59 -04:00
committed by GitHub
co-authored by Codex
parent 485421b1c3
commit 120ec4ebd2
28 changed files with 3124 additions and 2551 deletions
@@ -12,16 +12,15 @@ from cutlass import Float32, Int32, Uint32, Uint64
from quack.compile_utils import make_fake_tensor
from vllm.cute_utils import recast_val
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
CuTeDSLLaunchSpec,
VllmCuTeDSLJitKernel,
cutedsl_kernel_launcher,
)
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
kernel_launcher,
)
from vllm.triton_utils import tl, triton
@@ -569,7 +568,7 @@ class StableTopKFromGatheredCandidatesKernel(
)
return gathered, out
@cutedsl_kernel_launcher
@kernel_launcher
def __call__(
self,
gathered: torch.Tensor,
@@ -577,16 +576,11 @@ class StableTopKFromGatheredCandidatesKernel(
*,
topk: int,
) -> CuTeDSLLaunchSpec[CompileKey]:
compile_key = self.dispatch(topk=topk, num_candidates=gathered.shape[1])
return (
compile_key,
(gathered, out),
{
"gathered_shape": tuple(gathered.shape),
"out_shape": tuple(out.shape),
"topk": topk,
},
compile_key = self.dispatch(
topk=topk,
num_candidates=gathered.shape[1],
)
return compile_key, (gathered, out)
_PACK_DCP_TOPK_CANDIDATES_KERNEL = PackDCPTopkCandidatesKernel()
+23
View File
@@ -15,6 +15,7 @@ from collections.abc import Callable, Iterable, Iterator, Mapping
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass, fields
from functools import wraps
from typing import Any, Generic, TypeVar, cast
__all__ = [
@@ -23,6 +24,7 @@ __all__ = [
"WarmupIntRange",
"get_ast_full_name",
"get_function_source_node",
"kernel_launcher",
"zip_inputs",
]
@@ -30,6 +32,26 @@ __all__ = [
CompileKeyT = TypeVar("CompileKeyT")
def kernel_launcher(call_fn: Callable[..., Any]) -> Callable[..., Any]:
"""Delegate a declarative launch specification to its backend owner."""
signature = inspect.signature(call_fn)
@wraps(call_fn)
def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:
launch_spec = call_fn(self, *args, **kwargs)
if not self.bind_launch_inputs:
return self.launch(launch_spec, {})
bound = signature.bind(self, *args, **kwargs)
bound.apply_defaults()
inputs = {
name: value for name, value in bound.arguments.items() if name != "self"
}
return self.launch(launch_spec, inputs)
return wrapper
@dataclass(frozen=True)
class WarmupIntRange:
"""Expand integers with range semantics or a custom monotonic progression."""
@@ -549,6 +571,7 @@ class VllmJitKernel(Generic[CompileKeyT], ABC):
"""Kernel wrapper that owns dispatch, warmup keys, and compilation."""
CompileKey: type[CompileKeyT]
bind_launch_inputs = True
def __init__(self) -> None:
self._dispatch_trace = _trace_compile_key_dispatch(self.dispatch)
@@ -3,19 +3,25 @@
from abc import abstractmethod
from collections.abc import Callable, Mapping
from functools import wraps
from typing import Any, ClassVar, Generic, TypeAlias, TypeVar
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel
from vllm.model_executor.warmup.jit_warmup import (
VllmJitKernel,
)
DEFAULT_CUTEDSL_COMPILE_OPTIONS = "--enable-tvm-ffi"
CompileKeyT = TypeVar("CompileKeyT")
CuTeDSLLaunchSpec: TypeAlias = tuple[
CompileKeyT,
tuple[Any, ...],
Mapping[str, Any] | None,
]
CuTeDSLLaunchSpec: TypeAlias = (
tuple[CompileKeyT, tuple[Any, ...]]
| tuple[CompileKeyT, tuple[Any, ...], Any]
| tuple[
CompileKeyT,
tuple[Any, ...],
Any,
Callable[[], Any],
]
)
def cutedsl_fake_stream(*, use_tvm_ffi_env_stream: bool = True) -> Any:
@@ -44,6 +50,7 @@ class VllmCuTeDSLJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
"""CuTeDSL owner whose compiled executor is shared by warmup and runtime."""
kernel: ClassVar[Any]
bind_launch_inputs = False
@abstractmethod
def warmup_inputs(self, compile_key: CompileKeyT) -> tuple[Any, ...]:
@@ -58,23 +65,15 @@ class VllmCuTeDSLJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
*self.warmup_inputs(compile_key),
)
def cutedsl_kernel_launcher(
call_fn: Callable[..., CuTeDSLLaunchSpec[CompileKeyT]],
) -> Callable[..., Any]:
"""Invoke a cached CuTeDSL executor from a declarative ``__call__``."""
@wraps(call_fn)
def wrapper(
self: VllmCuTeDSLJitKernel[CompileKeyT],
*args: Any,
**kwargs: Any,
def launch(
self,
launch_spec: CuTeDSLLaunchSpec[CompileKeyT],
_inputs: Mapping[str, Any],
) -> Any:
compile_key, launch_args, runtime_context = call_fn(self, *args, **kwargs)
executor = self._get_or_compile(
compile_key,
runtime_context=runtime_context,
)
return executor(*launch_args)
return wrapper
# (compile_key, args), optionally followed by output and epilogue.
compile_key, launch_args = launch_spec[:2]
executor = self._get_or_compile(compile_key)
result = executor(*launch_args)
if len(launch_spec) == 4:
return launch_spec[3]()
return launch_spec[2] if len(launch_spec) == 3 else result
@@ -5,7 +5,7 @@ import inspect
from abc import abstractmethod
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from functools import cached_property, wraps
from functools import cached_property
from typing import Any, ClassVar, Generic, TypeVar, cast
from vllm.model_executor.warmup.jit_warmup import (
@@ -128,14 +128,16 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
def launch(
self,
grid: tuple[int, ...] | None,
launch_spec: LaunchSpec,
inputs: Mapping[str, Any],
/,
*,
kernel: Any = None,
**kwargs: Any,
) -> Any:
kernel = kernel if kernel is not None else self.kernel
if len(launch_spec) == 3:
grid, launch_kwargs, outputs = launch_spec
else:
grid, launch_kwargs = launch_spec
outputs = None
kwargs = dict(launch_kwargs)
kernel = kwargs.pop("kernel", None) or self.kernel
runtime_launcher = kwargs.pop("_runtime_launcher", None)
runtime_launcher_arg_count = kwargs.pop("_runtime_launcher_arg_count", 0)
for name, value in inputs.items():
@@ -151,52 +153,20 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
kwargs["launch_pdl"] = self._warming_compile_key.launch_pdl
warmup = getattr(kernel, "warmup", None)
assert warmup is not None
return warmup(grid=(1,), **kwargs)
warmup(grid=(1,), **kwargs)
return outputs
if grid is None:
return None
return outputs
if runtime_launcher is not None:
regular_args = [
kwargs.pop(name)
for name in self._kernel_arg_names[:runtime_launcher_arg_count]
]
return runtime_launcher(kernel, grid, *regular_args, **kwargs)
return kernel[grid](**kwargs)
def kernel_launcher(
call_fn: Callable[..., LaunchSpec],
) -> Callable[..., Any]:
"""Launch a Triton kernel from a declarative ``__call__`` specification.
``call_fn`` returns either ``(grid, launch_kwargs)`` or, when it allocates
its own outputs, ``(grid, launch_kwargs, outputs)``. The declared outputs are
returned to the caller. A ``grid`` of ``None`` skips the launch (e.g. an
empty-token batch) while still returning the outputs.
"""
signature = inspect.signature(call_fn)
@wraps(call_fn)
def wrapper(
self: VllmTritonJitKernel[Any],
*args: Any,
**kwargs: Any,
) -> Any:
spec = call_fn(self, *args, **kwargs)
if len(spec) == 3:
grid, launch_kwargs, outputs = spec
runtime_launcher(kernel, grid, *regular_args, **kwargs)
else:
grid, launch_kwargs = spec
outputs = None
bound = signature.bind(self, *args, **kwargs)
bound.apply_defaults()
inputs = {
name: value for name, value in bound.arguments.items() if name != "self"
}
self.launch(grid, inputs, **launch_kwargs)
kernel[grid](**kwargs)
return outputs
return wrapper
@dataclass(frozen=True)
class TritonPointerInputVariant: