[5/N][warmup][DSv4] Migrate NVIDIA CuTeDSL attention kernels (#53566)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Codex
parent
485421b1c3
commit
120ec4ebd2
@@ -12,16 +12,15 @@ from cutlass import Float32, Int32, Uint32, Uint64
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
from vllm.cute_utils import recast_val
|
||||
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
|
||||
from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
|
||||
CuTeDSLLaunchSpec,
|
||||
VllmCuTeDSLJitKernel,
|
||||
cutedsl_kernel_launcher,
|
||||
)
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
kernel_launcher,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
@@ -569,7 +568,7 @@ class StableTopKFromGatheredCandidatesKernel(
|
||||
)
|
||||
return gathered, out
|
||||
|
||||
@cutedsl_kernel_launcher
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
gathered: torch.Tensor,
|
||||
@@ -577,16 +576,11 @@ class StableTopKFromGatheredCandidatesKernel(
|
||||
*,
|
||||
topk: int,
|
||||
) -> CuTeDSLLaunchSpec[CompileKey]:
|
||||
compile_key = self.dispatch(topk=topk, num_candidates=gathered.shape[1])
|
||||
return (
|
||||
compile_key,
|
||||
(gathered, out),
|
||||
{
|
||||
"gathered_shape": tuple(gathered.shape),
|
||||
"out_shape": tuple(out.shape),
|
||||
"topk": topk,
|
||||
},
|
||||
compile_key = self.dispatch(
|
||||
topk=topk,
|
||||
num_candidates=gathered.shape[1],
|
||||
)
|
||||
return compile_key, (gathered, out)
|
||||
|
||||
|
||||
_PACK_DCP_TOPK_CANDIDATES_KERNEL = PackDCPTopkCandidatesKernel()
|
||||
|
||||
@@ -15,6 +15,7 @@ from collections.abc import Callable, Iterable, Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass, fields
|
||||
from functools import wraps
|
||||
from typing import Any, Generic, TypeVar, cast
|
||||
|
||||
__all__ = [
|
||||
@@ -23,6 +24,7 @@ __all__ = [
|
||||
"WarmupIntRange",
|
||||
"get_ast_full_name",
|
||||
"get_function_source_node",
|
||||
"kernel_launcher",
|
||||
"zip_inputs",
|
||||
]
|
||||
|
||||
@@ -30,6 +32,26 @@ __all__ = [
|
||||
CompileKeyT = TypeVar("CompileKeyT")
|
||||
|
||||
|
||||
def kernel_launcher(call_fn: Callable[..., Any]) -> Callable[..., Any]:
|
||||
"""Delegate a declarative launch specification to its backend owner."""
|
||||
signature = inspect.signature(call_fn)
|
||||
|
||||
@wraps(call_fn)
|
||||
def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:
|
||||
launch_spec = call_fn(self, *args, **kwargs)
|
||||
if not self.bind_launch_inputs:
|
||||
return self.launch(launch_spec, {})
|
||||
|
||||
bound = signature.bind(self, *args, **kwargs)
|
||||
bound.apply_defaults()
|
||||
inputs = {
|
||||
name: value for name, value in bound.arguments.items() if name != "self"
|
||||
}
|
||||
return self.launch(launch_spec, inputs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WarmupIntRange:
|
||||
"""Expand integers with range semantics or a custom monotonic progression."""
|
||||
@@ -549,6 +571,7 @@ class VllmJitKernel(Generic[CompileKeyT], ABC):
|
||||
"""Kernel wrapper that owns dispatch, warmup keys, and compilation."""
|
||||
|
||||
CompileKey: type[CompileKeyT]
|
||||
bind_launch_inputs = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._dispatch_trace = _trace_compile_key_dispatch(self.dispatch)
|
||||
|
||||
@@ -3,19 +3,25 @@
|
||||
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Callable, Mapping
|
||||
from functools import wraps
|
||||
from typing import Any, ClassVar, Generic, TypeAlias, TypeVar
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel
|
||||
from vllm.model_executor.warmup.jit_warmup import (
|
||||
VllmJitKernel,
|
||||
)
|
||||
|
||||
DEFAULT_CUTEDSL_COMPILE_OPTIONS = "--enable-tvm-ffi"
|
||||
|
||||
CompileKeyT = TypeVar("CompileKeyT")
|
||||
CuTeDSLLaunchSpec: TypeAlias = tuple[
|
||||
CompileKeyT,
|
||||
tuple[Any, ...],
|
||||
Mapping[str, Any] | None,
|
||||
]
|
||||
CuTeDSLLaunchSpec: TypeAlias = (
|
||||
tuple[CompileKeyT, tuple[Any, ...]]
|
||||
| tuple[CompileKeyT, tuple[Any, ...], Any]
|
||||
| tuple[
|
||||
CompileKeyT,
|
||||
tuple[Any, ...],
|
||||
Any,
|
||||
Callable[[], Any],
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def cutedsl_fake_stream(*, use_tvm_ffi_env_stream: bool = True) -> Any:
|
||||
@@ -44,6 +50,7 @@ class VllmCuTeDSLJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
"""CuTeDSL owner whose compiled executor is shared by warmup and runtime."""
|
||||
|
||||
kernel: ClassVar[Any]
|
||||
bind_launch_inputs = False
|
||||
|
||||
@abstractmethod
|
||||
def warmup_inputs(self, compile_key: CompileKeyT) -> tuple[Any, ...]:
|
||||
@@ -58,23 +65,15 @@ class VllmCuTeDSLJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
*self.warmup_inputs(compile_key),
|
||||
)
|
||||
|
||||
|
||||
def cutedsl_kernel_launcher(
|
||||
call_fn: Callable[..., CuTeDSLLaunchSpec[CompileKeyT]],
|
||||
) -> Callable[..., Any]:
|
||||
"""Invoke a cached CuTeDSL executor from a declarative ``__call__``."""
|
||||
|
||||
@wraps(call_fn)
|
||||
def wrapper(
|
||||
self: VllmCuTeDSLJitKernel[CompileKeyT],
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
def launch(
|
||||
self,
|
||||
launch_spec: CuTeDSLLaunchSpec[CompileKeyT],
|
||||
_inputs: Mapping[str, Any],
|
||||
) -> Any:
|
||||
compile_key, launch_args, runtime_context = call_fn(self, *args, **kwargs)
|
||||
executor = self._get_or_compile(
|
||||
compile_key,
|
||||
runtime_context=runtime_context,
|
||||
)
|
||||
return executor(*launch_args)
|
||||
|
||||
return wrapper
|
||||
# (compile_key, args), optionally followed by output and epilogue.
|
||||
compile_key, launch_args = launch_spec[:2]
|
||||
executor = self._get_or_compile(compile_key)
|
||||
result = executor(*launch_args)
|
||||
if len(launch_spec) == 4:
|
||||
return launch_spec[3]()
|
||||
return launch_spec[2] if len(launch_spec) == 3 else result
|
||||
|
||||
@@ -5,7 +5,7 @@ import inspect
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from functools import cached_property, wraps
|
||||
from functools import cached_property
|
||||
from typing import Any, ClassVar, Generic, TypeVar, cast
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import (
|
||||
@@ -128,14 +128,16 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
|
||||
def launch(
|
||||
self,
|
||||
grid: tuple[int, ...] | None,
|
||||
launch_spec: LaunchSpec,
|
||||
inputs: Mapping[str, Any],
|
||||
/,
|
||||
*,
|
||||
kernel: Any = None,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
kernel = kernel if kernel is not None else self.kernel
|
||||
if len(launch_spec) == 3:
|
||||
grid, launch_kwargs, outputs = launch_spec
|
||||
else:
|
||||
grid, launch_kwargs = launch_spec
|
||||
outputs = None
|
||||
kwargs = dict(launch_kwargs)
|
||||
kernel = kwargs.pop("kernel", None) or self.kernel
|
||||
runtime_launcher = kwargs.pop("_runtime_launcher", None)
|
||||
runtime_launcher_arg_count = kwargs.pop("_runtime_launcher_arg_count", 0)
|
||||
for name, value in inputs.items():
|
||||
@@ -151,52 +153,20 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
kwargs["launch_pdl"] = self._warming_compile_key.launch_pdl
|
||||
warmup = getattr(kernel, "warmup", None)
|
||||
assert warmup is not None
|
||||
return warmup(grid=(1,), **kwargs)
|
||||
warmup(grid=(1,), **kwargs)
|
||||
return outputs
|
||||
if grid is None:
|
||||
return None
|
||||
return outputs
|
||||
if runtime_launcher is not None:
|
||||
regular_args = [
|
||||
kwargs.pop(name)
|
||||
for name in self._kernel_arg_names[:runtime_launcher_arg_count]
|
||||
]
|
||||
return runtime_launcher(kernel, grid, *regular_args, **kwargs)
|
||||
return kernel[grid](**kwargs)
|
||||
|
||||
|
||||
def kernel_launcher(
|
||||
call_fn: Callable[..., LaunchSpec],
|
||||
) -> Callable[..., Any]:
|
||||
"""Launch a Triton kernel from a declarative ``__call__`` specification.
|
||||
|
||||
``call_fn`` returns either ``(grid, launch_kwargs)`` or, when it allocates
|
||||
its own outputs, ``(grid, launch_kwargs, outputs)``. The declared outputs are
|
||||
returned to the caller. A ``grid`` of ``None`` skips the launch (e.g. an
|
||||
empty-token batch) while still returning the outputs.
|
||||
"""
|
||||
signature = inspect.signature(call_fn)
|
||||
|
||||
@wraps(call_fn)
|
||||
def wrapper(
|
||||
self: VllmTritonJitKernel[Any],
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
spec = call_fn(self, *args, **kwargs)
|
||||
if len(spec) == 3:
|
||||
grid, launch_kwargs, outputs = spec
|
||||
runtime_launcher(kernel, grid, *regular_args, **kwargs)
|
||||
else:
|
||||
grid, launch_kwargs = spec
|
||||
outputs = None
|
||||
bound = signature.bind(self, *args, **kwargs)
|
||||
bound.apply_defaults()
|
||||
inputs = {
|
||||
name: value for name, value in bound.arguments.items() if name != "self"
|
||||
}
|
||||
self.launch(grid, inputs, **launch_kwargs)
|
||||
kernel[grid](**kwargs)
|
||||
return outputs
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TritonPointerInputVariant:
|
||||
|
||||
Reference in New Issue
Block a user