[2/N][warmup][DSv4] Migrate sequence and DCP kernels (#53564)
Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Codex
OpenAI Codex
Lucas Wilkinson
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ecd600d91e
commit
1713b9866a
@@ -13,6 +13,8 @@ steps:
|
||||
- vllm/model_executor/warmup
|
||||
- tests/model_executor
|
||||
- tests/model_executor/test_jit_warmup.py
|
||||
- tests/model_executor/test_jit_warmup_cutedsl_launcher.py
|
||||
- tests/model_executor/test_jit_warmup_triton_launcher.py
|
||||
- tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
|
||||
commands:
|
||||
- apt-get update && apt-get install -y curl libsodium23
|
||||
@@ -41,6 +43,8 @@ steps:
|
||||
- vllm/model_executor/warmup
|
||||
- tests/model_executor
|
||||
- tests/model_executor/test_jit_warmup.py
|
||||
- tests/model_executor/test_jit_warmup_cutedsl_launcher.py
|
||||
- tests/model_executor/test_jit_warmup_triton_launcher.py
|
||||
- tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
@@ -532,10 +532,12 @@ def test_runtime_cache_miss_compiles_and_caches_executor() -> None:
|
||||
def test_registry_records_only_inside_model_setup_context() -> None:
|
||||
registry = JitWarmupRegistry(_config())
|
||||
kernel = RecordingToyKernel()
|
||||
config = _config()
|
||||
|
||||
kernel.register_warmup(3, _config())
|
||||
with registry.activate():
|
||||
kernel.register_warmup(3, _config())
|
||||
kernel.register_warmup(3, config)
|
||||
kernel.register_warmup(3, config)
|
||||
|
||||
assert len(registry) == 1
|
||||
assert kernel.compiled == []
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from vllm.model_executor.warmup import jit_warmup_cutedsl_helper
|
||||
from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
|
||||
CuTeDSLLaunchSpec,
|
||||
VllmCuTeDSLJitKernel,
|
||||
cutedsl_kernel_launcher,
|
||||
)
|
||||
|
||||
|
||||
class _TestCuTeDSLKernel(VllmCuTeDSLJitKernel["_TestCuTeDSLKernel.CompileKey"]):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
variant: int
|
||||
|
||||
@staticmethod
|
||||
def kernel(compile_key: CompileKey) -> tuple[str, int]:
|
||||
return "entry", compile_key.variant
|
||||
|
||||
def dispatch(self, *, variant: int) -> CompileKey:
|
||||
return self.CompileKey(variant=variant)
|
||||
|
||||
def get_warmup_keys(self) -> list[CompileKey]:
|
||||
return self._trace_dispatch(self.dispatch)(variant=(1, 2))
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
|
||||
return (f"fake-{compile_key.variant}",)
|
||||
|
||||
@cutedsl_kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
payload: str,
|
||||
*,
|
||||
variant: int,
|
||||
) -> CuTeDSLLaunchSpec[CompileKey]:
|
||||
compile_key = self.dispatch(variant=variant)
|
||||
return compile_key, (payload,), {"variant": variant}
|
||||
|
||||
|
||||
def test_cutedsl_launcher_reuses_compiled_executor(monkeypatch) -> None:
|
||||
compile_calls: list[tuple[Any, tuple[Any, ...]]] = []
|
||||
launch_calls: list[tuple[int, tuple[Any, ...]]] = []
|
||||
|
||||
def compile_cutedsl(entry: tuple[str, int], *args: Any) -> Any:
|
||||
compile_calls.append((entry, args))
|
||||
variant = entry[1]
|
||||
|
||||
def executor(*runtime_args: Any) -> tuple[int, tuple[Any, ...]]:
|
||||
launch_calls.append((variant, runtime_args))
|
||||
return variant, runtime_args
|
||||
|
||||
return executor
|
||||
|
||||
monkeypatch.setattr(
|
||||
jit_warmup_cutedsl_helper,
|
||||
"compile_cutedsl",
|
||||
compile_cutedsl,
|
||||
)
|
||||
kernel = _TestCuTeDSLKernel()
|
||||
compile_key = kernel.CompileKey(variant=1)
|
||||
|
||||
kernel.compile(compile_key)
|
||||
kernel.compile(compile_key)
|
||||
assert compile_calls == [(("entry", 1), ("fake-1",))]
|
||||
|
||||
assert kernel("runtime", variant=1) == (1, ("runtime",))
|
||||
assert kernel("other", variant=2) == (2, ("other",))
|
||||
assert compile_calls == [
|
||||
(("entry", 1), ("fake-1",)),
|
||||
(("entry", 2), ("fake-2",)),
|
||||
]
|
||||
assert launch_calls == [(1, ("runtime",)), (2, ("other",))]
|
||||
@@ -0,0 +1,128 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
VllmTritonJitKernel,
|
||||
kernel_launcher,
|
||||
)
|
||||
|
||||
|
||||
class _FakeTritonKernel:
|
||||
arg_names = ("first", "second", "CONST")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.warmup_calls: list[dict[str, Any]] = []
|
||||
|
||||
def warmup(self, **kwargs: Any) -> None:
|
||||
self.warmup_calls.append(kwargs)
|
||||
|
||||
|
||||
class _TestTritonKernel(VllmTritonJitKernel["_TestTritonKernel.CompileKey"]):
|
||||
kernel = _FakeTritonKernel()
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
value: int
|
||||
|
||||
def dispatch(self, *, value: int) -> CompileKey:
|
||||
return self.CompileKey(value=value)
|
||||
|
||||
def get_warmup_keys(self) -> list[CompileKey]:
|
||||
return self._trace_dispatch(self.dispatch)(value=1)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
return dict(first="warmup", second=compile_key.value, runtime_launcher=None)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
first: str,
|
||||
second: int,
|
||||
runtime_launcher: Any,
|
||||
) -> LaunchSpec:
|
||||
return (2,), dict(
|
||||
CONST=7,
|
||||
_runtime_launcher=runtime_launcher,
|
||||
_runtime_launcher_arg_count=2,
|
||||
)
|
||||
|
||||
|
||||
def test_triton_launcher_supports_compile_and_runtime_adapters() -> None:
|
||||
owner = _TestTritonKernel()
|
||||
owner.kernel.warmup_calls.clear()
|
||||
|
||||
owner.compile(owner.CompileKey(value=1))
|
||||
assert owner.kernel.warmup_calls == [
|
||||
{"grid": (1,), "first": "warmup", "second": 1, "CONST": 7}
|
||||
]
|
||||
|
||||
runtime_calls: list[tuple[Any, ...]] = []
|
||||
|
||||
def runtime_launcher(kernel: Any, grid: Any, *args: Any, **kwargs: Any) -> None:
|
||||
runtime_calls.append((kernel, grid, args, kwargs))
|
||||
|
||||
owner("runtime", 2, runtime_launcher)
|
||||
assert runtime_calls == [(owner.kernel, (2,), ("runtime", 2), {"CONST": 7})]
|
||||
|
||||
|
||||
def test_triton_launcher_supports_cpu_function_wrappers() -> None:
|
||||
calls: list[tuple[Any, ...]] = []
|
||||
|
||||
def kernel(first: str, second: int, CONST: int) -> None:
|
||||
calls.append((first, second, CONST))
|
||||
|
||||
class FuncWrapper:
|
||||
def __init__(self) -> None:
|
||||
self.func = kernel
|
||||
|
||||
def __getitem__(self, _grid: Any) -> Any:
|
||||
return self.func
|
||||
|
||||
class TestCpuKernel(_TestTritonKernel):
|
||||
kernel: Any = FuncWrapper()
|
||||
|
||||
owner = TestCpuKernel()
|
||||
|
||||
owner("runtime", 2, None)
|
||||
assert calls == [("runtime", 2, 7)]
|
||||
|
||||
|
||||
def test_compute_slot_mapping_uses_named_launcher_inputs(monkeypatch) -> None:
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.worker.block_table import ComputeSlotMappingKernel
|
||||
|
||||
owner = ComputeSlotMappingKernel()
|
||||
compile_key = owner.CompileKey(
|
||||
kv_cache_block_size=16,
|
||||
blocks_per_kv_block=1,
|
||||
total_cp_world_size=2,
|
||||
total_cp_rank=1,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
block_table_stride=128,
|
||||
block_size=16,
|
||||
)
|
||||
launches: list[tuple[Any, ...]] = []
|
||||
|
||||
def launch(grid: Any, inputs: Any, **kwargs: Any) -> None:
|
||||
launches.append((grid, inputs, kwargs))
|
||||
|
||||
monkeypatch.setattr(owner, "launch", launch)
|
||||
owner.compile(compile_key)
|
||||
|
||||
grid, inputs, kwargs = launches[0]
|
||||
assert grid == (2,)
|
||||
assert inputs["num_tokens"] == 2
|
||||
assert inputs["block_table_stride"] == 128
|
||||
assert kwargs == {
|
||||
"KV_CACHE_BLOCK_SIZE": 16,
|
||||
"BLOCKS_PER_KV_BLOCK": 1,
|
||||
"TOTAL_CP_WORLD_SIZE": 2,
|
||||
"TOTAL_CP_RANK": 1,
|
||||
"CP_KV_CACHE_INTERLEAVE_SIZE": 1,
|
||||
"PAD_ID": PAD_SLOT_ID,
|
||||
"BLOCK_SIZE": owner.triton_block_size,
|
||||
}
|
||||
@@ -1,7 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from functools import cache
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
@@ -11,6 +12,17 @@ from cutlass import Float32, Int32, Uint32, Uint64
|
||||
from quack.compile_utils import make_fake_tensor
|
||||
|
||||
from vllm.cute_utils import recast_val
|
||||
from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
|
||||
CuTeDSLLaunchSpec,
|
||||
VllmCuTeDSLJitKernel,
|
||||
cutedsl_kernel_launcher,
|
||||
)
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
kernel_launcher,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@@ -25,9 +37,7 @@ def stable_topk_from_gathered_candidates_cutedsl(
|
||||
dtype=torch.int32,
|
||||
device=gathered.device,
|
||||
)
|
||||
StableTopKFromGatheredCandidatesKernel.compile(topk, gathered.shape[1])(
|
||||
gathered, out
|
||||
)
|
||||
_STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL(gathered, out, topk=topk)
|
||||
return out
|
||||
|
||||
|
||||
@@ -41,380 +51,543 @@ def pack_dcp_topk_candidates_cutedsl(
|
||||
row_starts: torch.Tensor | None,
|
||||
) -> None:
|
||||
topk = topk_indices.shape[1]
|
||||
grid = (topk_indices.shape[0], triton.cdiv(topk, 512))
|
||||
row_starts_arg = row_starts if row_starts is not None else topk_indices
|
||||
_pack_dcp_topk_candidates_triton_kernel[grid](
|
||||
_PACK_DCP_TOPK_CANDIDATES_KERNEL(
|
||||
logits,
|
||||
topk_indices,
|
||||
packed,
|
||||
row_starts_arg,
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
topk_indices.stride(0),
|
||||
topk_indices.stride(1),
|
||||
packed.stride(0),
|
||||
packed.stride(1),
|
||||
packed.stride(2),
|
||||
logits.shape[1],
|
||||
DCP_RANK=dcp_rank,
|
||||
DCP_WORLD_SIZE=dcp_world_size,
|
||||
CP_INTERLEAVE=cp_interleave,
|
||||
HAS_ROW_STARTS=row_starts is not None,
|
||||
TOPK=topk,
|
||||
BLOCK_SIZE=512,
|
||||
num_warps=8,
|
||||
logits_stride0=logits.stride(0),
|
||||
logits_stride1=logits.stride(1),
|
||||
topk_stride0=topk_indices.stride(0),
|
||||
topk_stride1=topk_indices.stride(1),
|
||||
packed_stride0=packed.stride(0),
|
||||
packed_stride1=packed.stride(1),
|
||||
packed_stride2=packed.stride(2),
|
||||
num_cols=logits.shape[1],
|
||||
dcp_rank=dcp_rank,
|
||||
dcp_world_size=dcp_world_size,
|
||||
cp_interleave=cp_interleave,
|
||||
has_row_starts=row_starts is not None,
|
||||
topk=topk,
|
||||
block_size=512,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pack_dcp_topk_candidates_triton_kernel(
|
||||
logits,
|
||||
topk_indices,
|
||||
packed,
|
||||
row_starts,
|
||||
logits_stride0: tl.constexpr,
|
||||
logits_stride1: tl.constexpr,
|
||||
topk_stride0: tl.constexpr,
|
||||
topk_stride1: tl.constexpr,
|
||||
packed_stride0: tl.constexpr,
|
||||
packed_stride1: tl.constexpr,
|
||||
packed_stride2: tl.constexpr,
|
||||
num_cols,
|
||||
DCP_RANK: tl.constexpr,
|
||||
DCP_WORLD_SIZE: tl.constexpr,
|
||||
CP_INTERLEAVE: tl.constexpr,
|
||||
HAS_ROW_STARTS: tl.constexpr,
|
||||
TOPK: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
class PackDCPTopkCandidatesKernel(
|
||||
VllmTritonJitKernel["PackDCPTopkCandidatesKernel.CompileKey"]
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
tile = tl.program_id(1)
|
||||
cols = tile * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = cols < TOPK
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
dcp_rank: int
|
||||
dcp_world_size: int
|
||||
cp_interleave: int
|
||||
has_row_starts: bool
|
||||
topk: int
|
||||
block_size: int
|
||||
|
||||
local_idx = tl.load(
|
||||
topk_indices + row * topk_stride0 + cols * topk_stride1,
|
||||
mask=mask,
|
||||
other=-1,
|
||||
# These scalars only describe runtime layouts and bounds; the constexpr
|
||||
# fields below own the launch geometry and algorithmic specialization.
|
||||
@staticmethod
|
||||
@triton.jit(
|
||||
do_not_specialize=[
|
||||
"logits_stride0",
|
||||
"logits_stride1",
|
||||
"topk_stride0",
|
||||
"topk_stride1",
|
||||
"packed_stride0",
|
||||
"packed_stride1",
|
||||
"packed_stride2",
|
||||
"num_cols",
|
||||
]
|
||||
)
|
||||
valid = local_idx >= 0
|
||||
safe_local_idx = tl.maximum(local_idx, 0)
|
||||
def kernel(
|
||||
logits,
|
||||
topk_indices,
|
||||
packed,
|
||||
row_starts,
|
||||
logits_stride0,
|
||||
logits_stride1,
|
||||
topk_stride0,
|
||||
topk_stride1,
|
||||
packed_stride0,
|
||||
packed_stride1,
|
||||
packed_stride2,
|
||||
num_cols,
|
||||
DCP_RANK: tl.constexpr,
|
||||
DCP_WORLD_SIZE: tl.constexpr,
|
||||
CP_INTERLEAVE: tl.constexpr,
|
||||
HAS_ROW_STARTS: tl.constexpr,
|
||||
TOPK: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
tile = tl.program_id(1)
|
||||
cols = tile * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask = cols < TOPK
|
||||
|
||||
row_start = 0
|
||||
if HAS_ROW_STARTS:
|
||||
row_start = tl.load(row_starts + row)
|
||||
local_idx = tl.load(
|
||||
topk_indices + row * topk_stride0 + cols * topk_stride1,
|
||||
mask=mask,
|
||||
other=-1,
|
||||
)
|
||||
valid = local_idx >= 0
|
||||
safe_local_idx = tl.maximum(local_idx, 0)
|
||||
|
||||
score_col = safe_local_idx + row_start
|
||||
score_col = tl.minimum(score_col, tl.maximum(num_cols - 1, 0))
|
||||
score = tl.load(
|
||||
logits + row * logits_stride0 + score_col * logits_stride1,
|
||||
mask=mask & valid,
|
||||
other=-float("inf"),
|
||||
)
|
||||
row_start = 0
|
||||
if HAS_ROW_STARTS:
|
||||
row_start = tl.load(row_starts + row)
|
||||
|
||||
global_id = (
|
||||
(safe_local_idx // CP_INTERLEAVE) * (DCP_WORLD_SIZE * CP_INTERLEAVE)
|
||||
+ DCP_RANK * CP_INTERLEAVE
|
||||
+ safe_local_idx % CP_INTERLEAVE
|
||||
)
|
||||
global_id = tl.where(valid, global_id, -1)
|
||||
score_col = safe_local_idx + row_start
|
||||
score_col = tl.minimum(score_col, tl.maximum(num_cols - 1, 0))
|
||||
score = tl.load(
|
||||
logits + row * logits_stride0 + score_col * logits_stride1,
|
||||
mask=mask & valid,
|
||||
other=-float("inf"),
|
||||
)
|
||||
|
||||
packed_base = packed + row * packed_stride0 + cols * packed_stride1
|
||||
tl.store(packed_base, score, mask=mask)
|
||||
tl.store(packed_base + packed_stride2, global_id.to(tl.float32), mask=mask)
|
||||
global_id = (
|
||||
(safe_local_idx // CP_INTERLEAVE) * (DCP_WORLD_SIZE * CP_INTERLEAVE)
|
||||
+ DCP_RANK * CP_INTERLEAVE
|
||||
+ safe_local_idx % CP_INTERLEAVE
|
||||
)
|
||||
global_id = tl.where(valid, global_id, -1)
|
||||
|
||||
packed_base = packed + row * packed_stride0 + cols * packed_stride1
|
||||
tl.store(packed_base, score, mask=mask)
|
||||
tl.store(packed_base + packed_stride2, global_id.to(tl.float32), mask=mask)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
has_row_starts: bool,
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
**compile_key_fields,
|
||||
has_row_starts=has_row_starts,
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
if dcp_world_size <= 1:
|
||||
return []
|
||||
cp_interleave = vllm_config.parallel_config.cp_kv_cache_interleave_size
|
||||
topk = vllm_config.model_config.hf_config.index_topk
|
||||
if topk <= 0:
|
||||
return []
|
||||
|
||||
try:
|
||||
from vllm.distributed.parallel_state import get_dcp_group
|
||||
|
||||
dcp_rank = get_dcp_group().rank_in_group
|
||||
except Exception:
|
||||
dcp_rank = 0
|
||||
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
dcp_rank=dcp_rank,
|
||||
dcp_world_size=dcp_world_size,
|
||||
cp_interleave=cp_interleave,
|
||||
has_row_starts=(False, True),
|
||||
topk=topk,
|
||||
block_size=512,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
fp32_ptr = TritonWarmupTensor(torch.float32)
|
||||
int32_ptr = TritonWarmupTensor(torch.int32)
|
||||
return dict(
|
||||
logits=fp32_ptr,
|
||||
topk_indices=TritonWarmupTensor(torch.int32, shape=(1, compile_key.topk)),
|
||||
packed=fp32_ptr,
|
||||
row_starts_arg=int32_ptr,
|
||||
logits_stride0=1,
|
||||
logits_stride1=1,
|
||||
topk_stride0=1,
|
||||
topk_stride1=1,
|
||||
packed_stride0=1,
|
||||
packed_stride1=1,
|
||||
packed_stride2=1,
|
||||
num_cols=1,
|
||||
dcp_rank=compile_key.dcp_rank,
|
||||
dcp_world_size=compile_key.dcp_world_size,
|
||||
cp_interleave=compile_key.cp_interleave,
|
||||
has_row_starts=compile_key.has_row_starts,
|
||||
topk=compile_key.topk,
|
||||
block_size=compile_key.block_size,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
logits: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
packed: torch.Tensor,
|
||||
row_starts_arg: torch.Tensor,
|
||||
*,
|
||||
logits_stride0: int,
|
||||
logits_stride1: int,
|
||||
topk_stride0: int,
|
||||
topk_stride1: int,
|
||||
packed_stride0: int,
|
||||
packed_stride1: int,
|
||||
packed_stride2: int,
|
||||
num_cols: int,
|
||||
dcp_rank: int,
|
||||
dcp_world_size: int,
|
||||
cp_interleave: int,
|
||||
has_row_starts: bool,
|
||||
topk: int,
|
||||
block_size: int,
|
||||
) -> LaunchSpec:
|
||||
grid = (topk_indices.shape[0], triton.cdiv(topk, block_size))
|
||||
return grid, dict(
|
||||
row_starts=row_starts_arg,
|
||||
DCP_RANK=dcp_rank,
|
||||
DCP_WORLD_SIZE=dcp_world_size,
|
||||
CP_INTERLEAVE=cp_interleave,
|
||||
HAS_ROW_STARTS=has_row_starts,
|
||||
TOPK=topk,
|
||||
BLOCK_SIZE=block_size,
|
||||
num_warps=8,
|
||||
)
|
||||
|
||||
|
||||
@cute.jit
|
||||
def _warp_scan_inclusive_i32(val: Int32, lane: Int32) -> Int32:
|
||||
for i in cutlass.range_constexpr(cute.arch.WARP_SIZE.bit_length() - 1):
|
||||
offset = 1 << i
|
||||
partial = cute.arch.shuffle_sync_up(val, offset=offset, mask_and_clamp=0)
|
||||
if lane >= offset:
|
||||
val += partial
|
||||
return val
|
||||
|
||||
|
||||
@cute.jit
|
||||
def _block_scan_inclusive_i32(
|
||||
val: Int32,
|
||||
lane: Int32,
|
||||
warp_id: Int32,
|
||||
warp_scratch: cute.Tensor,
|
||||
warps_per_block: int,
|
||||
) -> Int32:
|
||||
prefix = _warp_scan_inclusive_i32(val, lane)
|
||||
if lane == Int32(cute.arch.WARP_SIZE - 1):
|
||||
warp_scratch[0, warp_id] = prefix
|
||||
cute.arch.sync_threads()
|
||||
|
||||
if warp_id == Int32(0):
|
||||
warp_total = Int32(0)
|
||||
if lane < Int32(warps_per_block):
|
||||
warp_total = warp_scratch[0, lane]
|
||||
warp_prefix = _warp_scan_inclusive_i32(warp_total, lane)
|
||||
if lane < Int32(warps_per_block):
|
||||
warp_scratch[0, lane] = warp_prefix - warp_total
|
||||
cute.arch.sync_threads()
|
||||
|
||||
return prefix + warp_scratch[0, warp_id]
|
||||
|
||||
|
||||
class StableTopKFromGatheredCandidatesKernel:
|
||||
class StableTopKFromGatheredCandidatesKernel(
|
||||
VllmCuTeDSLJitKernel["StableTopKFromGatheredCandidatesKernel.CompileKey"]
|
||||
):
|
||||
tb_size = 512
|
||||
hist_bins = 2048
|
||||
radix_bits = (hist_bins - 1).bit_length()
|
||||
assert hist_bins == 1 << radix_bits
|
||||
key_bits = Uint64.width
|
||||
radix_passes = (key_bits + radix_bits - 1) // radix_bits
|
||||
final_radix_bits = key_bits - radix_bits * (radix_passes - 1)
|
||||
hist_chunks = (hist_bins + tb_size - 1) // tb_size
|
||||
warps_per_block = tb_size // cute.arch.WARP_SIZE
|
||||
|
||||
def __init__(self, topk: int, num_candidates: int):
|
||||
assert num_candidates % self.tb_size == 0, (
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
topk: int
|
||||
num_candidates: int
|
||||
|
||||
@staticmethod
|
||||
def kernel(compile_key: CompileKey) -> Any:
|
||||
tb_size = StableTopKFromGatheredCandidatesKernel.tb_size
|
||||
hist_bins = StableTopKFromGatheredCandidatesKernel.hist_bins
|
||||
radix_bits = StableTopKFromGatheredCandidatesKernel.radix_bits
|
||||
key_bits = StableTopKFromGatheredCandidatesKernel.key_bits
|
||||
radix_passes = StableTopKFromGatheredCandidatesKernel.radix_passes
|
||||
final_radix_bits = StableTopKFromGatheredCandidatesKernel.final_radix_bits
|
||||
hist_chunks = StableTopKFromGatheredCandidatesKernel.hist_chunks
|
||||
warps_per_block = StableTopKFromGatheredCandidatesKernel.warps_per_block
|
||||
topk = compile_key.topk
|
||||
assert hist_bins == 1 << radix_bits
|
||||
assert compile_key.num_candidates % tb_size == 0, (
|
||||
"StableTopKFromGatheredCandidatesKernel requires candidate count "
|
||||
f"to be a multiple of {self.tb_size}, got {num_candidates}"
|
||||
f"to be a multiple of {tb_size}, got {compile_key.num_candidates}"
|
||||
)
|
||||
self.topk = topk
|
||||
self.keys_per_thread = num_candidates // self.tb_size
|
||||
keys_per_thread = compile_key.num_candidates // tb_size
|
||||
|
||||
@cute.struct
|
||||
class SharedStorage:
|
||||
hist: cute.struct.MemRange[Int32, self.hist_bins]
|
||||
hist: cute.struct.MemRange[Int32, hist_bins]
|
||||
committed_count: cute.struct.MemRange[Int32, 1]
|
||||
running_count: cute.struct.MemRange[Int32, 1]
|
||||
threshold_bin: cute.struct.MemRange[Int32, 1]
|
||||
threshold_found: cute.struct.MemRange[Int32, 1]
|
||||
include_threshold_bin: cute.struct.MemRange[Int32, 1]
|
||||
prefix_s: cute.struct.Align[cute.struct.MemRange[Uint64, 1], 8]
|
||||
warp_totals: cute.struct.MemRange[Int32, self.warps_per_block]
|
||||
warp_totals: cute.struct.MemRange[Int32, warps_per_block]
|
||||
|
||||
self.shared_storage = SharedStorage
|
||||
shared_storage = SharedStorage
|
||||
|
||||
@cute.jit
|
||||
def __call__(
|
||||
self,
|
||||
gathered: cute.Tensor,
|
||||
out: cute.Tensor,
|
||||
stream: CUstream,
|
||||
):
|
||||
grid = (gathered.shape[0], 1, 1)
|
||||
self.kernel(gathered, out).launch(
|
||||
grid=grid,
|
||||
block=(self.tb_size, 1, 1),
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def _stable_key(self, score: Float32, token_id: Int32) -> Uint64:
|
||||
bits = recast_val(score, Uint32)
|
||||
mask = Uint32(0x80000000)
|
||||
if (bits & Uint32(0x80000000)) != Uint32(0):
|
||||
mask = Uint32(0xFFFFFFFF)
|
||||
score_key = Uint64(bits ^ mask) << Uint64(32)
|
||||
id_key = Uint64(~Uint32(token_id))
|
||||
key = score_key | id_key
|
||||
if token_id < Int32(0):
|
||||
key = Uint64(0)
|
||||
return key
|
||||
|
||||
@cute.jit
|
||||
def _prefix_matches(
|
||||
self,
|
||||
key: Uint64,
|
||||
prefix: Uint64,
|
||||
prefix_bits: Int32,
|
||||
):
|
||||
matches = prefix_bits == Int32(0)
|
||||
if prefix_bits != Int32(0):
|
||||
shift = Int32(self.key_bits) - prefix_bits
|
||||
matches = (key >> Uint64(shift)) == (prefix >> Uint64(shift))
|
||||
return matches
|
||||
|
||||
@cute.jit
|
||||
def _radix_pass(
|
||||
self,
|
||||
keys: cute.Tensor,
|
||||
output: cute.Tensor,
|
||||
storage,
|
||||
tid: Int32,
|
||||
step: Int32,
|
||||
bits: int,
|
||||
is_final_pass: bool,
|
||||
):
|
||||
hist_smem = storage.hist.get_tensor(cute.make_layout((self.hist_bins,)))
|
||||
committed_count_smem = storage.committed_count.data_ptr()
|
||||
running_count_smem = storage.running_count.data_ptr()
|
||||
threshold_bin_smem = storage.threshold_bin.data_ptr()
|
||||
threshold_found_smem = storage.threshold_found.data_ptr()
|
||||
include_threshold_bin_smem = storage.include_threshold_bin.data_ptr()
|
||||
prefix_smem = storage.prefix_s.data_ptr()
|
||||
warp_totals_smem = storage.warp_totals.get_tensor(
|
||||
cute.make_layout((1, self.warps_per_block))
|
||||
)
|
||||
|
||||
prefix_bits = step * Int32(self.radix_bits)
|
||||
num_bins = 1 << bits
|
||||
block_scan_iterations = (num_bins + self.tb_size - 1) // self.tb_size
|
||||
shift = Int32(self.key_bits) - prefix_bits - Int32(bits)
|
||||
bin_mask = Uint64(num_bins - 1)
|
||||
prefix = prefix_smem.load()
|
||||
|
||||
for chunk in cutlass.range_constexpr(self.hist_chunks):
|
||||
hist_smem[tid + Int32(chunk * self.tb_size)] = Int32(0)
|
||||
if tid == Int32(0):
|
||||
running_count_smem.store(committed_count_smem.load())
|
||||
include_threshold_bin_smem.store(Int32(0))
|
||||
threshold_found_smem.store(Int32(0))
|
||||
cute.arch.sync_threads()
|
||||
|
||||
for key_idx in cutlass.range_constexpr(self.keys_per_thread):
|
||||
key = keys[key_idx]
|
||||
if self._prefix_matches(key, prefix, prefix_bits):
|
||||
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
|
||||
cute.arch.atomic_add(
|
||||
hist_smem.iterator + bin_idx,
|
||||
Int32(1),
|
||||
sem="relaxed",
|
||||
scope="cta",
|
||||
@cute.jit
|
||||
def warp_scan_inclusive_i32(val: Int32, lane: Int32) -> Int32:
|
||||
for i in cutlass.range_constexpr(cute.arch.WARP_SIZE.bit_length() - 1):
|
||||
offset = 1 << i
|
||||
partial = cute.arch.shuffle_sync_up(
|
||||
val, offset=offset, mask_and_clamp=0
|
||||
)
|
||||
cute.arch.sync_threads()
|
||||
if lane >= offset:
|
||||
val += partial
|
||||
return val
|
||||
|
||||
lane = cute.arch.lane_idx()
|
||||
warp_id = cute.arch.warp_idx()
|
||||
# Each iteration scans one tb_size-wide slice of bins, high to low.
|
||||
iter = Int32(0)
|
||||
threshold_found = threshold_found_smem.load()
|
||||
while threshold_found == Int32(0) and iter < Int32(block_scan_iterations):
|
||||
bin_idx = Int32(num_bins - 1) - (iter * Int32(self.tb_size) + tid)
|
||||
count = hist_smem[bin_idx]
|
||||
chunk_inclusive = _block_scan_inclusive_i32(
|
||||
count,
|
||||
lane,
|
||||
warp_id,
|
||||
warp_totals_smem,
|
||||
self.warps_per_block,
|
||||
@cute.jit
|
||||
def block_scan_inclusive_i32(
|
||||
val: Int32,
|
||||
lane: Int32,
|
||||
warp_id: Int32,
|
||||
warp_scratch: cute.Tensor,
|
||||
warps_per_block: int,
|
||||
) -> Int32:
|
||||
prefix = warp_scan_inclusive_i32(val, lane)
|
||||
if lane == Int32(cute.arch.WARP_SIZE - 1):
|
||||
warp_scratch[0, warp_id] = prefix
|
||||
cute.arch.sync_threads()
|
||||
|
||||
if warp_id == Int32(0):
|
||||
warp_total = Int32(0)
|
||||
if lane < Int32(warps_per_block):
|
||||
warp_total = warp_scratch[0, lane]
|
||||
warp_prefix = warp_scan_inclusive_i32(warp_total, lane)
|
||||
if lane < Int32(warps_per_block):
|
||||
warp_scratch[0, lane] = warp_prefix - warp_total
|
||||
cute.arch.sync_threads()
|
||||
|
||||
return prefix + warp_scratch[0, warp_id]
|
||||
|
||||
@cute.jit
|
||||
def stable_key(score: Float32, token_id: Int32) -> Uint64:
|
||||
bits = recast_val(score, Uint32)
|
||||
mask = Uint32(0x80000000)
|
||||
if (bits & Uint32(0x80000000)) != Uint32(0):
|
||||
mask = Uint32(0xFFFFFFFF)
|
||||
score_key = Uint64(bits ^ mask) << Uint64(32)
|
||||
id_key = Uint64(~Uint32(token_id))
|
||||
key = score_key | id_key
|
||||
if token_id < Int32(0):
|
||||
key = Uint64(0)
|
||||
return key
|
||||
|
||||
@cute.jit
|
||||
def prefix_matches(
|
||||
key: Uint64,
|
||||
prefix: Uint64,
|
||||
prefix_bits: Int32,
|
||||
):
|
||||
matches = prefix_bits == Int32(0)
|
||||
if prefix_bits != Int32(0):
|
||||
shift = Int32(key_bits) - prefix_bits
|
||||
matches = (key >> Uint64(shift)) == (prefix >> Uint64(shift))
|
||||
return matches
|
||||
|
||||
@cute.jit
|
||||
def radix_pass(
|
||||
keys: cute.Tensor,
|
||||
output: cute.Tensor,
|
||||
storage,
|
||||
tid: Int32,
|
||||
step: Int32,
|
||||
bits: int,
|
||||
is_final_pass: bool,
|
||||
):
|
||||
hist_smem = storage.hist.get_tensor(cute.make_layout((hist_bins,)))
|
||||
committed_count_smem = storage.committed_count.data_ptr()
|
||||
running_count_smem = storage.running_count.data_ptr()
|
||||
threshold_bin_smem = storage.threshold_bin.data_ptr()
|
||||
threshold_found_smem = storage.threshold_found.data_ptr()
|
||||
include_threshold_bin_smem = storage.include_threshold_bin.data_ptr()
|
||||
prefix_smem = storage.prefix_s.data_ptr()
|
||||
warp_totals_smem = storage.warp_totals.get_tensor(
|
||||
cute.make_layout((1, warps_per_block))
|
||||
)
|
||||
running_count = running_count_smem.load()
|
||||
prior_in_scan_slice = chunk_inclusive - count
|
||||
remaining = Int32(self.topk) - running_count - prior_in_scan_slice
|
||||
if count > Int32(0) and remaining > Int32(0) and remaining <= count:
|
||||
threshold_bin_smem.store(bin_idx)
|
||||
if count <= remaining or cutlass.const_expr(is_final_pass):
|
||||
include_threshold_bin_smem.store(Int32(1))
|
||||
threshold_found_smem.store(Int32(1))
|
||||
# Barrier: every thread must finish reading running_count for this
|
||||
# slice before tb_size-1 advances it, else a warp racing ahead to
|
||||
# the store makes a lagging thread double-count the slice total
|
||||
# (-> remaining too small -> threshold too high -> under-fill).
|
||||
cute.arch.sync_threads()
|
||||
if tid == Int32(self.tb_size - 1):
|
||||
running_count_smem.store(running_count + chunk_inclusive)
|
||||
|
||||
prefix_bits = step * Int32(radix_bits)
|
||||
num_bins = 1 << bits
|
||||
block_scan_iterations = (num_bins + tb_size - 1) // tb_size
|
||||
shift = Int32(key_bits) - prefix_bits - Int32(bits)
|
||||
bin_mask = Uint64(num_bins - 1)
|
||||
prefix = prefix_smem.load()
|
||||
|
||||
for chunk in cutlass.range_constexpr(hist_chunks):
|
||||
hist_smem[tid + Int32(chunk * tb_size)] = Int32(0)
|
||||
if tid == Int32(0):
|
||||
running_count_smem.store(committed_count_smem.load())
|
||||
include_threshold_bin_smem.store(Int32(0))
|
||||
threshold_found_smem.store(Int32(0))
|
||||
cute.arch.sync_threads()
|
||||
|
||||
threshold_found = threshold_found_smem.load()
|
||||
iter += Int32(1)
|
||||
|
||||
threshold = threshold_bin_smem.load()
|
||||
should_include_threshold = include_threshold_bin_smem.load() != Int32(0)
|
||||
for key_idx in cutlass.range_constexpr(self.keys_per_thread):
|
||||
key = keys[key_idx]
|
||||
if self._prefix_matches(key, prefix, prefix_bits):
|
||||
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
|
||||
selected = bin_idx > threshold
|
||||
if should_include_threshold:
|
||||
selected = selected or bin_idx == threshold
|
||||
if selected:
|
||||
dst = cute.arch.atomic_add(
|
||||
committed_count_smem,
|
||||
for key_idx in cutlass.range_constexpr(keys_per_thread):
|
||||
key = keys[key_idx]
|
||||
if prefix_matches(key, prefix, prefix_bits):
|
||||
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
|
||||
cute.arch.atomic_add(
|
||||
hist_smem.iterator + bin_idx,
|
||||
Int32(1),
|
||||
sem="relaxed",
|
||||
scope="cta",
|
||||
)
|
||||
if dst < Int32(self.topk):
|
||||
output[dst] = recast_val(~Uint32(key), Int32)
|
||||
cute.arch.sync_threads()
|
||||
cute.arch.sync_threads()
|
||||
|
||||
pass_finished = include_threshold_bin_smem.load()
|
||||
if tid == Int32(0) and pass_finished == Int32(0):
|
||||
prefix_smem.store(prefix | (Uint64(threshold) << Uint64(shift)))
|
||||
cute.arch.sync_threads()
|
||||
return pass_finished
|
||||
lane = cute.arch.lane_idx()
|
||||
warp_id = cute.arch.warp_idx()
|
||||
# Each iteration scans one tb_size-wide slice of bins, high to low.
|
||||
iter = Int32(0)
|
||||
threshold_found = threshold_found_smem.load()
|
||||
while threshold_found == Int32(0) and iter < Int32(block_scan_iterations):
|
||||
bin_idx = Int32(num_bins - 1) - (iter * Int32(tb_size) + tid)
|
||||
count = hist_smem[bin_idx]
|
||||
chunk_inclusive = block_scan_inclusive_i32(
|
||||
count,
|
||||
lane,
|
||||
warp_id,
|
||||
warp_totals_smem,
|
||||
warps_per_block,
|
||||
)
|
||||
running_count = running_count_smem.load()
|
||||
prior_in_scan_slice = chunk_inclusive - count
|
||||
remaining = Int32(topk) - running_count - prior_in_scan_slice
|
||||
if count > Int32(0) and remaining > Int32(0) and remaining <= count:
|
||||
threshold_bin_smem.store(bin_idx)
|
||||
if count <= remaining or cutlass.const_expr(is_final_pass):
|
||||
include_threshold_bin_smem.store(Int32(1))
|
||||
threshold_found_smem.store(Int32(1))
|
||||
# Barrier: every thread must finish reading running_count for this
|
||||
# slice before tb_size-1 advances it, else a warp racing ahead to
|
||||
# the store makes a lagging thread double-count the slice total
|
||||
# (-> remaining too small -> threshold too high -> under-fill).
|
||||
cute.arch.sync_threads()
|
||||
if tid == Int32(tb_size - 1):
|
||||
running_count_smem.store(running_count + chunk_inclusive)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
@cute.kernel
|
||||
def kernel(
|
||||
threshold_found = threshold_found_smem.load()
|
||||
iter += Int32(1)
|
||||
|
||||
threshold = threshold_bin_smem.load()
|
||||
should_include_threshold = include_threshold_bin_smem.load() != Int32(0)
|
||||
for key_idx in cutlass.range_constexpr(keys_per_thread):
|
||||
key = keys[key_idx]
|
||||
if prefix_matches(key, prefix, prefix_bits):
|
||||
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
|
||||
selected = bin_idx > threshold
|
||||
if should_include_threshold:
|
||||
selected = selected or bin_idx == threshold
|
||||
if selected:
|
||||
dst = cute.arch.atomic_add(
|
||||
committed_count_smem,
|
||||
Int32(1),
|
||||
sem="relaxed",
|
||||
scope="cta",
|
||||
)
|
||||
if dst < Int32(topk):
|
||||
output[dst] = recast_val(~Uint32(key), Int32)
|
||||
cute.arch.sync_threads()
|
||||
|
||||
pass_finished = include_threshold_bin_smem.load()
|
||||
if tid == Int32(0) and pass_finished == Int32(0):
|
||||
prefix_smem.store(prefix | (Uint64(threshold) << Uint64(shift)))
|
||||
cute.arch.sync_threads()
|
||||
return pass_finished
|
||||
|
||||
@cute.kernel
|
||||
def device_kernel(input: cute.Tensor, out: cute.Tensor):
|
||||
row, _, _ = cute.arch.block_idx()
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
input_row = input[row, None, None]
|
||||
output_row = out[row, None]
|
||||
keys = cute.make_rmem_tensor((keys_per_thread,), Uint64)
|
||||
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
storage = smem.allocate(shared_storage, 8)
|
||||
committed_count_smem = storage.committed_count.data_ptr()
|
||||
prefix_smem = storage.prefix_s.data_ptr()
|
||||
for i in range(tid, topk, tb_size):
|
||||
output_row[i] = Int32(-1)
|
||||
|
||||
for key_idx in cutlass.range_constexpr(keys_per_thread):
|
||||
col = tid + Int32(key_idx * tb_size)
|
||||
score = Float32(input_row[col, 0])
|
||||
token_id = Int32(input_row[col, 1])
|
||||
keys[key_idx] = stable_key(score, token_id)
|
||||
|
||||
if tid == Int32(0):
|
||||
committed_count_smem.store(Int32(0))
|
||||
prefix_smem.store(Uint64(0))
|
||||
cute.arch.sync_threads()
|
||||
|
||||
step = Int32(0)
|
||||
finished = Int32(0)
|
||||
while finished == Int32(0) and step < Int32(radix_passes - 1):
|
||||
finished = radix_pass(
|
||||
keys,
|
||||
output_row,
|
||||
storage,
|
||||
tid,
|
||||
step,
|
||||
radix_bits,
|
||||
False,
|
||||
)
|
||||
step += Int32(1)
|
||||
|
||||
if finished == Int32(0):
|
||||
radix_pass(
|
||||
keys,
|
||||
output_row,
|
||||
storage,
|
||||
tid,
|
||||
Int32(radix_passes - 1),
|
||||
final_radix_bits,
|
||||
True,
|
||||
)
|
||||
|
||||
@cute.jit
|
||||
def host_entrypoint(
|
||||
gathered: cute.Tensor,
|
||||
out: cute.Tensor,
|
||||
stream: CUstream,
|
||||
):
|
||||
grid = (gathered.shape[0], 1, 1)
|
||||
device_kernel(gathered, out).launch(
|
||||
grid=grid,
|
||||
block=(tb_size, 1, 1),
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
return host_entrypoint
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
input: cute.Tensor,
|
||||
out: cute.Tensor,
|
||||
):
|
||||
row, _, _ = cute.arch.block_idx()
|
||||
tid, _, _ = cute.arch.thread_idx()
|
||||
input_row = input[row, None, None]
|
||||
output_row = out[row, None]
|
||||
keys = cute.make_rmem_tensor((self.keys_per_thread,), Uint64)
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(**compile_key_fields)
|
||||
|
||||
smem = cutlass.utils.SmemAllocator()
|
||||
storage = smem.allocate(self.shared_storage, 8)
|
||||
committed_count_smem = storage.committed_count.data_ptr()
|
||||
prefix_smem = storage.prefix_s.data_ptr()
|
||||
for i in range(tid, self.topk, self.tb_size):
|
||||
output_row[i] = Int32(-1)
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
if dcp_world_size <= 1:
|
||||
return []
|
||||
topk = vllm_config.model_config.hf_config.index_topk
|
||||
if topk <= 0:
|
||||
return []
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
topk=topk,
|
||||
num_candidates=topk * dcp_world_size,
|
||||
)
|
||||
|
||||
for key_idx in cutlass.range_constexpr(self.keys_per_thread):
|
||||
col = tid + Int32(key_idx * self.tb_size)
|
||||
score = Float32(input_row[col, 0])
|
||||
token_id = Int32(input_row[col, 1])
|
||||
keys[key_idx] = self._stable_key(score, token_id)
|
||||
|
||||
if tid == Int32(0):
|
||||
committed_count_smem.store(Int32(0))
|
||||
prefix_smem.store(Uint64(0))
|
||||
cute.arch.sync_threads()
|
||||
|
||||
step = Int32(0)
|
||||
finished = Int32(0)
|
||||
while finished == Int32(0) and step < Int32(self.radix_passes - 1):
|
||||
finished = self._radix_pass(
|
||||
keys,
|
||||
output_row,
|
||||
storage,
|
||||
tid,
|
||||
step,
|
||||
self.radix_bits,
|
||||
False,
|
||||
)
|
||||
step += Int32(1)
|
||||
|
||||
if finished == Int32(0):
|
||||
self._radix_pass(
|
||||
keys,
|
||||
output_row,
|
||||
storage,
|
||||
tid,
|
||||
Int32(self.radix_passes - 1),
|
||||
self.final_radix_bits,
|
||||
True,
|
||||
)
|
||||
|
||||
@cache
|
||||
@staticmethod
|
||||
def compile(topk: int, num_candidates: int):
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
|
||||
num_rows = cute.sym_int()
|
||||
|
||||
gathered = cute.runtime.make_fake_tensor(
|
||||
Float32,
|
||||
(num_rows, num_candidates, 2),
|
||||
(num_rows, compile_key.num_candidates, 2),
|
||||
stride=(cute.sym_int64(divisibility=2), 2, 1),
|
||||
assumed_align=8,
|
||||
)
|
||||
out = make_fake_tensor(Int32, (num_rows, topk), divisibility=1)
|
||||
|
||||
kernel = StableTopKFromGatheredCandidatesKernel(topk, num_candidates)
|
||||
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
|
||||
return cute.compile(
|
||||
kernel,
|
||||
gathered,
|
||||
out,
|
||||
stream,
|
||||
options="--enable-tvm-ffi",
|
||||
out = make_fake_tensor(
|
||||
Int32,
|
||||
(num_rows, compile_key.topk),
|
||||
divisibility=1,
|
||||
)
|
||||
return gathered, out
|
||||
|
||||
@cutedsl_kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
gathered: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
*,
|
||||
topk: int,
|
||||
) -> CuTeDSLLaunchSpec[CompileKey]:
|
||||
compile_key = self.dispatch(topk=topk, num_candidates=gathered.shape[1])
|
||||
return (
|
||||
compile_key,
|
||||
(gathered, out),
|
||||
{
|
||||
"gathered_shape": tuple(gathered.shape),
|
||||
"out_shape": tuple(out.shape),
|
||||
"topk": topk,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
_PACK_DCP_TOPK_CANDIDATES_KERNEL = PackDCPTopkCandidatesKernel()
|
||||
_STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL = StableTopKFromGatheredCandidatesKernel()
|
||||
|
||||
@@ -768,7 +768,8 @@ class SparseAttnIndexer(CustomOp):
|
||||
# DCP scalars are constant for the run; resolve them here (config is set
|
||||
# during model construction) and pass them into the custom op, rather
|
||||
# than threading them through per-step metadata.
|
||||
parallel_config = get_current_vllm_config().parallel_config
|
||||
vllm_config = get_current_vllm_config()
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self._parallel_config = parallel_config
|
||||
self.dcp_world_size = parallel_config.decode_context_parallel_size
|
||||
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
|
||||
@@ -780,6 +781,28 @@ class SparseAttnIndexer(CustomOp):
|
||||
"the current vLLM environment."
|
||||
)
|
||||
|
||||
if vllm_config.kernel_config.enable_jit_warmup:
|
||||
from vllm.v1.attention.ops.common import (
|
||||
_PACK_SEQ_TRITON_KERNEL,
|
||||
_UNPACK_SEQ_TRITON_KERNEL,
|
||||
)
|
||||
|
||||
pack_dtype = torch.uint8 if use_fp4_cache else current_platform.fp8_dtype()
|
||||
_PACK_SEQ_TRITON_KERNEL.register_warmup(
|
||||
dtype=pack_dtype,
|
||||
pad_value=0 if use_fp4_cache else -float("inf"),
|
||||
)
|
||||
_UNPACK_SEQ_TRITON_KERNEL.register_warmup()
|
||||
|
||||
if self.dcp_world_size > 1 and current_platform.is_cuda() and has_cutedsl():
|
||||
from vllm.model_executor.kernels.attention.dsa.dcp_indexer_cutedsl import ( # noqa: E501
|
||||
_PACK_DCP_TOPK_CANDIDATES_KERNEL,
|
||||
_STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL,
|
||||
)
|
||||
|
||||
_PACK_DCP_TOPK_CANDIDATES_KERNEL.register_warmup()
|
||||
_STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL.register_warmup()
|
||||
|
||||
@property
|
||||
def cp_kv_cache_interleave_size(self) -> int:
|
||||
"""With PD+DCP, the real value isn't known until block_size is finalized,
|
||||
|
||||
@@ -670,6 +670,34 @@ class VllmJitKernel(Generic[CompileKeyT], ABC):
|
||||
self.compile(compile_key)
|
||||
|
||||
|
||||
def _same_value(left: Any, right: Any) -> bool:
|
||||
"""True if two registration values are the same object or compare equal.
|
||||
|
||||
Identity is checked first so shared singletons (e.g. ``vllm_config``, torch
|
||||
dtypes) short-circuit before any potentially deep or non-boolean ``__eq__``.
|
||||
"""
|
||||
if left is right:
|
||||
return True
|
||||
try:
|
||||
return bool(left == right)
|
||||
except (TypeError, ValueError, RuntimeError):
|
||||
return False
|
||||
|
||||
|
||||
def _same_registration(
|
||||
left: tuple[tuple[Any, ...], dict[str, Any]],
|
||||
right: tuple[tuple[Any, ...], dict[str, Any]],
|
||||
) -> bool:
|
||||
"""True if two ``(args, kwargs)`` registrations are equivalent."""
|
||||
left_args, left_kwargs = left
|
||||
right_args, right_kwargs = right
|
||||
if len(left_args) != len(right_args) or left_kwargs.keys() != right_kwargs.keys():
|
||||
return False
|
||||
return all(
|
||||
_same_value(x, y) for x, y in zip(left_args, right_args, strict=True)
|
||||
) and all(_same_value(left_kwargs[k], right_kwargs[k]) for k in left_kwargs)
|
||||
|
||||
|
||||
class JitWarmupRegistry:
|
||||
"""Collect and compile JIT kernels selected during runner setup."""
|
||||
|
||||
@@ -713,13 +741,15 @@ class JitWarmupRegistry:
|
||||
kwargs: dict[str, Any],
|
||||
) -> None:
|
||||
registrations = self._registrations.setdefault(kernel, [])
|
||||
if (
|
||||
not args
|
||||
and not kwargs
|
||||
and any(
|
||||
not registered_args and not registered_kwargs
|
||||
for registered_args, registered_kwargs in registrations
|
||||
)
|
||||
# Every layer of a deep model appends an identical (args, kwargs) here
|
||||
# (e.g. a 61-layer DSA model registers the same pack (dtype, pad_value)
|
||||
# 61 times); each expands to the same compile keys, so tracing
|
||||
# get_warmup_keys once per distinct registration is sufficient. Dedup on
|
||||
# identity-or-equality -- identity short-circuits shared singletons like
|
||||
# vllm_config before any deep __eq__.
|
||||
if any(
|
||||
_same_registration(registered, (args, kwargs))
|
||||
for registered in registrations
|
||||
):
|
||||
return
|
||||
registrations.append((args, kwargs))
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Callable, Mapping
|
||||
from functools import wraps
|
||||
from typing import Any, ClassVar, Generic, TypeAlias, TypeVar
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel
|
||||
|
||||
DEFAULT_CUTEDSL_COMPILE_OPTIONS = "--enable-tvm-ffi"
|
||||
|
||||
CompileKeyT = TypeVar("CompileKeyT")
|
||||
CuTeDSLLaunchSpec: TypeAlias = tuple[
|
||||
CompileKeyT,
|
||||
tuple[Any, ...],
|
||||
Mapping[str, Any] | None,
|
||||
]
|
||||
|
||||
|
||||
def cutedsl_fake_stream(*, use_tvm_ffi_env_stream: bool = True) -> Any:
|
||||
import cutlass.cute as cute
|
||||
|
||||
return cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=use_tvm_ffi_env_stream)
|
||||
|
||||
|
||||
def compile_cutedsl(
|
||||
entry: Callable[..., Any],
|
||||
*args: Any,
|
||||
options: str = DEFAULT_CUTEDSL_COMPILE_OPTIONS,
|
||||
use_tvm_ffi_env_stream: bool = True,
|
||||
) -> Any:
|
||||
import cutlass.cute as cute
|
||||
|
||||
return cute.compile(
|
||||
entry,
|
||||
*args,
|
||||
cutedsl_fake_stream(use_tvm_ffi_env_stream=use_tvm_ffi_env_stream),
|
||||
options=options,
|
||||
)
|
||||
|
||||
|
||||
class VllmCuTeDSLJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
"""CuTeDSL owner whose compiled executor is shared by warmup and runtime."""
|
||||
|
||||
kernel: ClassVar[Any]
|
||||
|
||||
@abstractmethod
|
||||
def warmup_inputs(self, compile_key: CompileKeyT) -> tuple[Any, ...]:
|
||||
"""Return fake arguments that compile one executor specialization."""
|
||||
raise NotImplementedError
|
||||
|
||||
def compile(self, compile_key: CompileKeyT) -> None:
|
||||
if compile_key in self._compiled_cache:
|
||||
return
|
||||
self._compiled_cache[compile_key] = compile_cutedsl(
|
||||
self.kernel(compile_key),
|
||||
*self.warmup_inputs(compile_key),
|
||||
)
|
||||
|
||||
|
||||
def cutedsl_kernel_launcher(
|
||||
call_fn: Callable[..., CuTeDSLLaunchSpec[CompileKeyT]],
|
||||
) -> Callable[..., Any]:
|
||||
"""Invoke a cached CuTeDSL executor from a declarative ``__call__``."""
|
||||
|
||||
@wraps(call_fn)
|
||||
def wrapper(
|
||||
self: VllmCuTeDSLJitKernel[CompileKeyT],
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
compile_key, launch_args, runtime_context = call_fn(self, *args, **kwargs)
|
||||
executor = self._get_or_compile(
|
||||
compile_key,
|
||||
runtime_context=runtime_context,
|
||||
)
|
||||
return executor(*launch_args)
|
||||
|
||||
return wrapper
|
||||
@@ -106,8 +106,16 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
self._warming = False
|
||||
|
||||
@cached_property
|
||||
def _kernel_param_names(self) -> frozenset[str]:
|
||||
return frozenset(self.kernel.arg_names)
|
||||
def _kernel_arg_names(self) -> tuple[str, ...]:
|
||||
arg_names = getattr(self.kernel, "arg_names", None)
|
||||
if arg_names is not None:
|
||||
return tuple(arg_names)
|
||||
wrapped = getattr(self.kernel, "func", None)
|
||||
if wrapped is not None:
|
||||
return tuple(inspect.signature(wrapped).parameters)
|
||||
raise TypeError(
|
||||
f"Cannot inspect kernel parameters for {type(self.kernel).__name__}"
|
||||
)
|
||||
|
||||
def launch(
|
||||
self,
|
||||
@@ -116,14 +124,22 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
/,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
runtime_launcher = kwargs.pop("_runtime_launcher", None)
|
||||
runtime_launcher_arg_count = kwargs.pop("_runtime_launcher_arg_count", 0)
|
||||
for name, value in inputs.items():
|
||||
target = name if name in self._kernel_param_names else f"{name}_ptr"
|
||||
if target in self._kernel_param_names and target not in kwargs:
|
||||
target = name if name in self._kernel_arg_names else f"{name}_ptr"
|
||||
if target in self._kernel_arg_names and target not in kwargs:
|
||||
kwargs[target] = value
|
||||
if self._warming:
|
||||
warmup = getattr(self.kernel, "warmup", None)
|
||||
assert warmup is not None
|
||||
return warmup(grid=(1,), **kwargs)
|
||||
if runtime_launcher is not None:
|
||||
regular_args = [
|
||||
kwargs.pop(name)
|
||||
for name in self._kernel_arg_names[:runtime_launcher_arg_count]
|
||||
]
|
||||
return runtime_launcher(self.kernel, grid, *regular_args, **kwargs)
|
||||
return self.kernel[grid](**kwargs)
|
||||
|
||||
|
||||
|
||||
+259
-103
@@ -1,63 +1,157 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
kernel_launcher,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pack_seq_kernel(
|
||||
x_ptr, # [N, D]
|
||||
out_ptr, # [B, Lmax, D]
|
||||
lengths_ptr, # *i32, [B]
|
||||
N: tl.constexpr,
|
||||
D: tl.constexpr,
|
||||
Lmax: tl.constexpr,
|
||||
PAD_VALUE: tl.constexpr,
|
||||
PAD_IS_UINT8: tl.constexpr,
|
||||
BLOCK_T: tl.constexpr, # timesteps per program
|
||||
BLOCK_D: tl.constexpr, # features per program
|
||||
):
|
||||
pid_b = tl.program_id(0) # batch id
|
||||
pid_t = tl.program_id(1) # block over time dimension
|
||||
pid_d = tl.program_id(2) # block over feature dimension
|
||||
off_t = pid_t * BLOCK_T + tl.arange(0, BLOCK_T) # [BLOCK_T]
|
||||
off_d = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) # [BLOCK_D]
|
||||
class PackSeqTritonKernel(VllmTritonJitKernel["PackSeqTritonKernel.CompileKey"]):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
dtype: torch.dtype
|
||||
pad_value: float | int
|
||||
pad_is_uint8: bool
|
||||
block_t: int
|
||||
block_d: int
|
||||
|
||||
# Compute start index and sequence length from cumulative lengths
|
||||
in_start = 0
|
||||
for i in range(pid_b):
|
||||
in_start += tl.load(lengths_ptr + i)
|
||||
seq_len = tl.load(lengths_ptr + pid_b)
|
||||
# N is unused; D and Lmax only affect runtime addressing and masks. The
|
||||
# compile-time tile shape remains controlled by BLOCK_T and BLOCK_D.
|
||||
@staticmethod
|
||||
@triton.jit(do_not_specialize=["N", "D", "Lmax"])
|
||||
def kernel(
|
||||
x_ptr, # [N, D]
|
||||
out_ptr, # [B, Lmax, D]
|
||||
lengths_ptr, # *i32, [B]
|
||||
N,
|
||||
D,
|
||||
Lmax,
|
||||
PAD_VALUE: tl.constexpr,
|
||||
PAD_IS_UINT8: tl.constexpr,
|
||||
BLOCK_T: tl.constexpr, # timesteps per program
|
||||
BLOCK_D: tl.constexpr, # features per program
|
||||
):
|
||||
pid_b = tl.program_id(0) # batch id
|
||||
pid_t = tl.program_id(1) # block over time dimension
|
||||
pid_d = tl.program_id(2) # block over feature dimension
|
||||
off_t = pid_t * BLOCK_T + tl.arange(0, BLOCK_T) # [BLOCK_T]
|
||||
off_d = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) # [BLOCK_D]
|
||||
|
||||
# valid time positions for this block
|
||||
t_mask = off_t < Lmax
|
||||
# Compute start index and sequence length from cumulative lengths
|
||||
in_start = 0
|
||||
for i in range(pid_b):
|
||||
in_start += tl.load(lengths_ptr + i)
|
||||
seq_len = tl.load(lengths_ptr + pid_b)
|
||||
|
||||
# compute input row indices for valid (b, t)
|
||||
in_row = in_start + off_t
|
||||
valid_row = (off_t < seq_len) & t_mask
|
||||
# valid time positions for this block
|
||||
t_mask = off_t < Lmax
|
||||
|
||||
# Pointers
|
||||
# x_ptr: row-major [N, D]
|
||||
x_row_ptr = x_ptr + in_row[:, None] * D + off_d[None, :]
|
||||
# compute input row indices for valid (b, t)
|
||||
in_row = in_start + off_t
|
||||
valid_row = (off_t < seq_len) & t_mask
|
||||
|
||||
# out_ptr: row-major [B, Lmax, D]
|
||||
out_row_ptr = out_ptr + (pid_b * Lmax + off_t)[:, None] * D + off_d[None, :]
|
||||
# Pointers
|
||||
# x_ptr: row-major [N, D]
|
||||
x_row_ptr = x_ptr + in_row[:, None] * D + off_d[None, :]
|
||||
|
||||
# Initialize with PAD. PAD_IS_UINT8 selects the pad tensor's dtype so
|
||||
# integer-typed outputs (e.g. MXFP4 packed nibbles, ue8m0 scale bytes)
|
||||
# get an exact-byte pad rather than going through an fp32→uint8 cast
|
||||
# that's implementation-defined outside of value 0.
|
||||
d_mask = off_d[None, :] < D
|
||||
if PAD_IS_UINT8:
|
||||
pad_vals = tl.full([BLOCK_T, BLOCK_D], PAD_VALUE, tl.uint8)
|
||||
else:
|
||||
pad_vals = tl.full([BLOCK_T, BLOCK_D], PAD_VALUE, tl.float32)
|
||||
tl.store(out_row_ptr, pad_vals, mask=t_mask[:, None] & d_mask)
|
||||
# out_ptr: row-major [B, Lmax, D]
|
||||
out_row_ptr = out_ptr + (pid_b * Lmax + off_t)[:, None] * D + off_d[None, :]
|
||||
|
||||
# Load & write only where within seq_len
|
||||
x_vals = tl.load(x_row_ptr, mask=valid_row[:, None] & d_mask)
|
||||
tl.store(out_row_ptr, x_vals, mask=valid_row[:, None] & d_mask)
|
||||
# Initialize with PAD. PAD_IS_UINT8 selects the pad tensor's dtype so
|
||||
# integer-typed outputs (e.g. MXFP4 packed nibbles, ue8m0 scale bytes)
|
||||
# get an exact-byte pad rather than going through an fp32→uint8 cast
|
||||
# that's implementation-defined outside of value 0.
|
||||
d_mask = off_d[None, :] < D
|
||||
if PAD_IS_UINT8:
|
||||
pad_vals = tl.full([BLOCK_T, BLOCK_D], PAD_VALUE, tl.uint8)
|
||||
else:
|
||||
pad_vals = tl.full([BLOCK_T, BLOCK_D], PAD_VALUE, tl.float32)
|
||||
tl.store(out_row_ptr, pad_vals, mask=t_mask[:, None] & d_mask)
|
||||
|
||||
# Load & write only where within seq_len
|
||||
x_vals = tl.load(x_row_ptr, mask=valid_row[:, None] & d_mask)
|
||||
tl.store(out_row_ptr, x_vals, mask=valid_row[:, None] & d_mask)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
pad_value: float | int,
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
is_uint8 = dtype == torch.uint8
|
||||
return self.CompileKey(
|
||||
**compile_key_fields,
|
||||
dtype=dtype,
|
||||
pad_value=int(pad_value) if is_uint8 else float(pad_value),
|
||||
pad_is_uint8=is_uint8,
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
pad_value: float | int,
|
||||
) -> list[CompileKey]:
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
dtype=dtype,
|
||||
pad_value=pad_value,
|
||||
block_t=64,
|
||||
block_d=64,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
return dict(
|
||||
x_reshaped=TritonWarmupTensor(compile_key.dtype),
|
||||
out=TritonWarmupTensor(compile_key.dtype),
|
||||
lengths=TritonWarmupTensor(torch.int32),
|
||||
N=1,
|
||||
D=1,
|
||||
Lmax=1,
|
||||
pad_value=compile_key.pad_value,
|
||||
pad_is_uint8=compile_key.pad_is_uint8,
|
||||
block_t=compile_key.block_t,
|
||||
block_d=compile_key.block_d,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
x_reshaped: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
lengths: torch.Tensor,
|
||||
*,
|
||||
N: int,
|
||||
D: int,
|
||||
Lmax: int,
|
||||
pad_value: float | int,
|
||||
pad_is_uint8: bool,
|
||||
block_t: int,
|
||||
block_d: int,
|
||||
) -> LaunchSpec:
|
||||
grid = (
|
||||
lengths.shape[0],
|
||||
triton.cdiv(Lmax, block_t),
|
||||
triton.cdiv(D, block_d),
|
||||
)
|
||||
return grid, dict(
|
||||
x_ptr=x_reshaped,
|
||||
lengths_ptr=lengths,
|
||||
PAD_VALUE=pad_value,
|
||||
PAD_IS_UINT8=pad_is_uint8,
|
||||
BLOCK_T=block_t,
|
||||
BLOCK_D=block_d,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
)
|
||||
|
||||
|
||||
def pack_seq_triton(
|
||||
@@ -109,20 +203,18 @@ def pack_seq_triton(
|
||||
|
||||
out = torch.empty((B, Lmax, D), device=x.device, dtype=x.dtype)
|
||||
|
||||
grid = (B, triton.cdiv(Lmax, block_t), triton.cdiv(D, block_d))
|
||||
_pack_seq_kernel[grid](
|
||||
lengths = lengths.int()
|
||||
_PACK_SEQ_TRITON_KERNEL(
|
||||
x_reshaped,
|
||||
out,
|
||||
lengths.int(),
|
||||
N,
|
||||
D,
|
||||
Lmax,
|
||||
PAD_VALUE=pad_constexpr,
|
||||
PAD_IS_UINT8=is_uint8,
|
||||
BLOCK_T=block_t,
|
||||
BLOCK_D=block_d,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
lengths,
|
||||
N=N,
|
||||
D=D,
|
||||
Lmax=Lmax,
|
||||
pad_value=pad_constexpr,
|
||||
pad_is_uint8=is_uint8,
|
||||
block_t=block_t,
|
||||
block_d=block_d,
|
||||
)
|
||||
|
||||
if len(original_shape) > 2:
|
||||
@@ -131,47 +223,109 @@ def pack_seq_triton(
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _unpack_seq_triton_kernel(
|
||||
packed_ptr, # [B, Lmax, D]
|
||||
out_ptr, # [N, D]
|
||||
lengths_ptr, # *i32, [B]
|
||||
B: tl.constexpr,
|
||||
Lmax: tl.constexpr,
|
||||
D: tl.constexpr,
|
||||
BLOCK_T: tl.constexpr, # timesteps per program
|
||||
BLOCK_D: tl.constexpr, # features per program
|
||||
):
|
||||
pid_b = tl.program_id(0) # batch id
|
||||
pid_t = tl.program_id(1) # block over time dimension
|
||||
pid_d = tl.program_id(2) # block over feature dimension
|
||||
off_t = pid_t * BLOCK_T + tl.arange(0, BLOCK_T) # [BLOCK_T]
|
||||
off_d = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) # [BLOCK_D]
|
||||
class UnpackSeqTritonKernel(VllmTritonJitKernel["UnpackSeqTritonKernel.CompileKey"]):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
dtype: torch.dtype
|
||||
block_t: int
|
||||
block_d: int
|
||||
|
||||
# bounds: compute start from cumulative lengths
|
||||
in_start = 0
|
||||
for i in range(pid_b):
|
||||
in_start += tl.load(lengths_ptr + i)
|
||||
seq_len = tl.load(lengths_ptr + pid_b)
|
||||
# B, Lmax, and D only affect runtime addressing and masks. The compile-time
|
||||
# tile shape remains controlled by BLOCK_T and BLOCK_D.
|
||||
@staticmethod
|
||||
@triton.jit(do_not_specialize=["B", "Lmax", "D"])
|
||||
def kernel(
|
||||
packed_ptr, # [B, Lmax, D]
|
||||
out_ptr, # [N, D]
|
||||
lengths_ptr, # *i32, [B]
|
||||
B,
|
||||
Lmax,
|
||||
D,
|
||||
BLOCK_T: tl.constexpr, # timesteps per program
|
||||
BLOCK_D: tl.constexpr, # features per program
|
||||
):
|
||||
pid_b = tl.program_id(0) # batch id
|
||||
pid_t = tl.program_id(1) # block over time dimension
|
||||
pid_d = tl.program_id(2) # block over feature dimension
|
||||
off_t = pid_t * BLOCK_T + tl.arange(0, BLOCK_T) # [BLOCK_T]
|
||||
off_d = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) # [BLOCK_D]
|
||||
|
||||
# valid time positions for this block
|
||||
t_mask = off_t < Lmax
|
||||
valid_row = (off_t < seq_len) & t_mask
|
||||
# bounds: compute start from cumulative lengths
|
||||
in_start = 0
|
||||
for i in range(pid_b):
|
||||
in_start += tl.load(lengths_ptr + i)
|
||||
seq_len = tl.load(lengths_ptr + pid_b)
|
||||
|
||||
# compute output row indices for valid (b, t)
|
||||
out_row = in_start + off_t
|
||||
# valid time positions for this block
|
||||
t_mask = off_t < Lmax
|
||||
valid_row = (off_t < seq_len) & t_mask
|
||||
|
||||
# Pointers
|
||||
# packed_ptr: row-major [B, Lmax, D]
|
||||
packed_row_ptr = packed_ptr + (pid_b * Lmax + off_t)[:, None] * D + off_d[None, :]
|
||||
# compute output row indices for valid (b, t)
|
||||
out_row = in_start + off_t
|
||||
|
||||
# out_ptr: row-major [N, D]
|
||||
out_row_ptr = out_ptr + out_row[:, None] * D + off_d[None, :]
|
||||
# Pointers
|
||||
# packed_ptr: row-major [B, Lmax, D]
|
||||
packed_row_ptr = (
|
||||
packed_ptr + (pid_b * Lmax + off_t)[:, None] * D + off_d[None, :]
|
||||
)
|
||||
|
||||
# Load from packed tensor and store to output
|
||||
d_mask = off_d[None, :] < D
|
||||
packed_vals = tl.load(packed_row_ptr, mask=valid_row[:, None] & d_mask)
|
||||
tl.store(out_row_ptr, packed_vals, mask=valid_row[:, None] & d_mask)
|
||||
# out_ptr: row-major [N, D]
|
||||
out_row_ptr = out_ptr + out_row[:, None] * D + off_d[None, :]
|
||||
|
||||
# Load from packed tensor and store to output
|
||||
d_mask = off_d[None, :] < D
|
||||
packed_vals = tl.load(packed_row_ptr, mask=valid_row[:, None] & d_mask)
|
||||
tl.store(out_row_ptr, packed_vals, mask=valid_row[:, None] & d_mask)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(**compile_key_fields, dtype=dtype)
|
||||
|
||||
def get_warmup_keys(self) -> list[CompileKey]:
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
dtype=torch.int32,
|
||||
block_t=64,
|
||||
block_d=64,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
return dict(
|
||||
packed_reshaped=TritonWarmupTensor(compile_key.dtype),
|
||||
out=TritonWarmupTensor(compile_key.dtype),
|
||||
lengths=TritonWarmupTensor(torch.int32),
|
||||
B=1,
|
||||
Lmax=1,
|
||||
D=1,
|
||||
block_t=compile_key.block_t,
|
||||
block_d=compile_key.block_d,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
packed_reshaped: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
lengths: torch.Tensor,
|
||||
*,
|
||||
B: int,
|
||||
Lmax: int,
|
||||
D: int,
|
||||
block_t: int,
|
||||
block_d: int,
|
||||
) -> LaunchSpec:
|
||||
grid = (B, triton.cdiv(Lmax, block_t), triton.cdiv(D, block_d))
|
||||
return grid, dict(
|
||||
packed_ptr=packed_reshaped,
|
||||
lengths_ptr=lengths,
|
||||
BLOCK_T=block_t,
|
||||
BLOCK_D=block_d,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
)
|
||||
|
||||
|
||||
def unpack_seq_triton(
|
||||
@@ -209,18 +363,16 @@ def unpack_seq_triton(
|
||||
|
||||
out = torch.empty((N, D), device=packed_tensor.device, dtype=packed_tensor.dtype)
|
||||
|
||||
grid = (B, triton.cdiv(Lmax, block_t), triton.cdiv(D, block_d))
|
||||
_unpack_seq_triton_kernel[grid](
|
||||
lengths = lengths.int()
|
||||
_UNPACK_SEQ_TRITON_KERNEL(
|
||||
packed_reshaped,
|
||||
out,
|
||||
lengths.int(),
|
||||
B,
|
||||
Lmax,
|
||||
D,
|
||||
BLOCK_T=block_t,
|
||||
BLOCK_D=block_d,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
lengths,
|
||||
B=B,
|
||||
Lmax=Lmax,
|
||||
D=D,
|
||||
block_t=block_t,
|
||||
block_d=block_d,
|
||||
)
|
||||
|
||||
# Reshape output back to original dimensions (except first dimension)
|
||||
@@ -229,3 +381,7 @@ def unpack_seq_triton(
|
||||
out = out.reshape(output_shape)
|
||||
|
||||
return out
|
||||
|
||||
|
||||
_PACK_SEQ_TRITON_KERNEL = PackSeqTritonKernel()
|
||||
_UNPACK_SEQ_TRITON_KERNEL = UnpackSeqTritonKernel()
|
||||
|
||||
+269
-97
@@ -6,7 +6,8 @@ from __future__ import annotations
|
||||
|
||||
import functools
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Protocol
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
@@ -15,6 +16,16 @@ import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_dcp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.jit_warmup import (
|
||||
WarmupIntRange,
|
||||
)
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
kernel_launcher,
|
||||
triton_scalar_specialization_rep,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.ops.cp_common import (
|
||||
DirectCPWorkspace,
|
||||
@@ -65,93 +76,254 @@ def mask_dcp_empty_shards_(
|
||||
# AG + RS/AR implementation
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _correct_attn_cp_out_kernel(
|
||||
outputs_ptr,
|
||||
new_output_ptr,
|
||||
lses_ptr,
|
||||
vlse_ptr,
|
||||
outputs_stride_B,
|
||||
outputs_stride_H,
|
||||
outputs_stride_D,
|
||||
lses_stride_N,
|
||||
lses_stride_B,
|
||||
lses_stride_H,
|
||||
lse_idx,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
N_ROUNDED: tl.constexpr,
|
||||
IS_BASE_E: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Apply the all-gathered lses to correct each local rank's attention
|
||||
output. we still need perform a cross-rank reduction to obtain the
|
||||
final attention output.
|
||||
class CorrectAttnCPOutKernel(VllmTritonJitKernel["CorrectAttnCPOutKernel.CompileKey"]):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
output_dtype: torch.dtype
|
||||
lse_dtype: torch.dtype
|
||||
outputs_stride_b: int
|
||||
outputs_stride_h: int
|
||||
outputs_stride_d: int
|
||||
lses_stride_n: int
|
||||
lses_stride_b: int
|
||||
lses_stride_h: int
|
||||
lse_idx: int
|
||||
head_dim: int
|
||||
n_rounded: int
|
||||
is_base_e: bool
|
||||
|
||||
Args:
|
||||
outputs_ptr (triton.PointerType):
|
||||
Pointer to input tensor of shape [ B, H, D ]
|
||||
lses_ptr (triton.PointerType):
|
||||
Pointer to input tensor of shape [ N, B, H ]
|
||||
new_output_ptr (triton.PointerType):
|
||||
Pointer to output tensor of shape [ B, H, D ]
|
||||
vlse_ptr (triton.PointerType):
|
||||
Pointer to output tensor of shape [ B, H ]
|
||||
"""
|
||||
batch_idx = tl.program_id(axis=0).to(tl.int64)
|
||||
head_idx = tl.program_id(axis=1).to(tl.int64)
|
||||
d_offsets = tl.arange(0, HEAD_DIM)
|
||||
num_n_offsets = tl.arange(0, N_ROUNDED)
|
||||
@staticmethod
|
||||
@triton.jit
|
||||
def kernel(
|
||||
outputs_ptr,
|
||||
new_output_ptr,
|
||||
lses_ptr,
|
||||
vlse_ptr,
|
||||
outputs_stride_B,
|
||||
outputs_stride_H,
|
||||
outputs_stride_D,
|
||||
lses_stride_N,
|
||||
lses_stride_B,
|
||||
lses_stride_H,
|
||||
lse_idx,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
N_ROUNDED: tl.constexpr,
|
||||
IS_BASE_E: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Apply the all-gathered lses to correct each local rank's attention
|
||||
output. we still need perform a cross-rank reduction to obtain the
|
||||
final attention output.
|
||||
|
||||
# shape = [N]
|
||||
lse_offsets = (
|
||||
num_n_offsets * lses_stride_N
|
||||
+ batch_idx * lses_stride_B
|
||||
+ head_idx * lses_stride_H
|
||||
)
|
||||
Args:
|
||||
outputs_ptr (triton.PointerType):
|
||||
Pointer to input tensor of shape [ B, H, D ]
|
||||
lses_ptr (triton.PointerType):
|
||||
Pointer to input tensor of shape [ N, B, H ]
|
||||
new_output_ptr (triton.PointerType):
|
||||
Pointer to output tensor of shape [ B, H, D ]
|
||||
vlse_ptr (triton.PointerType):
|
||||
Pointer to output tensor of shape [ B, H ]
|
||||
"""
|
||||
batch_idx = tl.program_id(axis=0).to(tl.int64)
|
||||
head_idx = tl.program_id(axis=1).to(tl.int64)
|
||||
d_offsets = tl.arange(0, HEAD_DIM)
|
||||
num_n_offsets = tl.arange(0, N_ROUNDED)
|
||||
|
||||
# calc final lse
|
||||
lse = tl.load(lses_ptr + lse_offsets).to(tl.float32)
|
||||
lse = tl.where((lse != lse) | (lse == float("inf")), -float("inf"), lse)
|
||||
lse_max = tl.max(lse, axis=0)
|
||||
lse_max = tl.where(lse_max == -float("inf"), 0, lse_max)
|
||||
lse -= lse_max
|
||||
if IS_BASE_E:
|
||||
lse_exp = tl.exp(lse)
|
||||
lse_acc = tl.sum(lse_exp, axis=0)
|
||||
lse = tl.log(lse_acc)
|
||||
else:
|
||||
lse_exp = tl.exp2(lse)
|
||||
lse_acc = tl.sum(lse_exp, axis=0)
|
||||
lse = tl.log2(lse_acc)
|
||||
lse += lse_max
|
||||
# shape = [N]
|
||||
lse_offsets = (
|
||||
num_n_offsets * lses_stride_N
|
||||
+ batch_idx * lses_stride_B
|
||||
+ head_idx * lses_stride_H
|
||||
)
|
||||
|
||||
lse_offsets = batch_idx * lses_stride_B + head_idx * lses_stride_H
|
||||
tl.store(vlse_ptr + lse_offsets, lse)
|
||||
# calc final lse
|
||||
lse = tl.load(lses_ptr + lse_offsets).to(tl.float32)
|
||||
lse = tl.where((lse != lse) | (lse == float("inf")), -float("inf"), lse)
|
||||
lse_max = tl.max(lse, axis=0)
|
||||
lse_max = tl.where(lse_max == -float("inf"), 0, lse_max)
|
||||
lse -= lse_max
|
||||
if IS_BASE_E:
|
||||
lse_exp = tl.exp(lse)
|
||||
lse_acc = tl.sum(lse_exp, axis=0)
|
||||
lse = tl.log(lse_acc)
|
||||
else:
|
||||
lse_exp = tl.exp2(lse)
|
||||
lse_acc = tl.sum(lse_exp, axis=0)
|
||||
lse = tl.log2(lse_acc)
|
||||
lse += lse_max
|
||||
|
||||
# shape = [D]
|
||||
output_offsets = (
|
||||
batch_idx * outputs_stride_B
|
||||
+ head_idx * outputs_stride_H
|
||||
+ d_offsets * outputs_stride_D
|
||||
)
|
||||
lse_offsets = batch_idx * lses_stride_B + head_idx * lses_stride_H
|
||||
tl.store(vlse_ptr + lse_offsets, lse)
|
||||
|
||||
# correct output
|
||||
lse_offset = (
|
||||
lse_idx * lses_stride_N + batch_idx * lses_stride_B + head_idx * lses_stride_H
|
||||
)
|
||||
lse_tmp = tl.load(lses_ptr + lse_offset).to(tl.float32)
|
||||
lse_finally = lse_tmp - lse
|
||||
lse_finally = tl.where(
|
||||
(lse_finally != lse_finally) | (lse_finally == float("inf")),
|
||||
-float("inf"),
|
||||
lse_finally,
|
||||
)
|
||||
factor = tl.exp(lse_finally) if IS_BASE_E else tl.exp2(lse_finally)
|
||||
output = tl.load(outputs_ptr + output_offsets)
|
||||
output = output * factor
|
||||
output = tl.where(factor == 0.0, 0.0, output)
|
||||
# shape = [D]
|
||||
output_offsets = (
|
||||
batch_idx * outputs_stride_B
|
||||
+ head_idx * outputs_stride_H
|
||||
+ d_offsets * outputs_stride_D
|
||||
)
|
||||
|
||||
tl.store(new_output_ptr + output_offsets, output)
|
||||
# correct output
|
||||
lse_offset = (
|
||||
lse_idx * lses_stride_N
|
||||
+ batch_idx * lses_stride_B
|
||||
+ head_idx * lses_stride_H
|
||||
)
|
||||
lse_tmp = tl.load(lses_ptr + lse_offset).to(tl.float32)
|
||||
lse_finally = lse_tmp - lse
|
||||
lse_finally = tl.where(
|
||||
(lse_finally != lse_finally) | (lse_finally == float("inf")),
|
||||
-float("inf"),
|
||||
lse_finally,
|
||||
)
|
||||
factor = tl.exp(lse_finally) if IS_BASE_E else tl.exp2(lse_finally)
|
||||
output = tl.load(outputs_ptr + output_offsets)
|
||||
output = output * factor
|
||||
output = tl.where(factor == 0.0, 0.0, output)
|
||||
|
||||
tl.store(new_output_ptr + output_offsets, output)
|
||||
|
||||
def dispatch( # type: ignore[override]
|
||||
self,
|
||||
*,
|
||||
output_dtype: torch.dtype,
|
||||
lse_dtype: torch.dtype,
|
||||
num_tokens: int,
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
n_rounded: int,
|
||||
lse_idx: int,
|
||||
is_base_e: bool,
|
||||
) -> CompileKey:
|
||||
# Warm the contiguous [B, H, D] output and [N, B, H] LSE layouts used
|
||||
# by the production collective path. Runtime still forwards real
|
||||
# strides so unsupported layouts remain correct, but may compile on
|
||||
# first use under the JIT monitor.
|
||||
outputs_stride_d = 1
|
||||
outputs_stride_h = head_dim
|
||||
outputs_stride_b = num_heads * head_dim
|
||||
lses_stride_h = 1
|
||||
lses_stride_b = num_heads
|
||||
lses_stride_n = num_tokens * num_heads
|
||||
return self.CompileKey(
|
||||
output_dtype=output_dtype,
|
||||
lse_dtype=lse_dtype,
|
||||
outputs_stride_b=triton_scalar_specialization_rep(outputs_stride_b),
|
||||
outputs_stride_h=triton_scalar_specialization_rep(outputs_stride_h),
|
||||
outputs_stride_d=triton_scalar_specialization_rep(outputs_stride_d),
|
||||
lses_stride_n=triton_scalar_specialization_rep(lses_stride_n),
|
||||
lses_stride_b=triton_scalar_specialization_rep(lses_stride_b),
|
||||
lses_stride_h=triton_scalar_specialization_rep(lses_stride_h),
|
||||
lse_idx=triton_scalar_specialization_rep(lse_idx),
|
||||
head_dim=head_dim,
|
||||
n_rounded=n_rounded,
|
||||
is_base_e=is_base_e,
|
||||
)
|
||||
|
||||
def get_warmup_keys(
|
||||
self,
|
||||
vllm_config: Any,
|
||||
*,
|
||||
output_dtype: torch.dtype | None = None,
|
||||
num_heads: int | None = None,
|
||||
head_dim: int | None = None,
|
||||
is_base_e: bool | tuple[bool, ...] = (False, True),
|
||||
) -> list[CompileKey]:
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
get_mla_dims,
|
||||
)
|
||||
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
max_tokens = min(
|
||||
16,
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
)
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
if num_heads is None:
|
||||
num_heads = (
|
||||
hf_config.num_attention_heads
|
||||
// vllm_config.parallel_config.tensor_parallel_size
|
||||
)
|
||||
if head_dim is None:
|
||||
head_dim = get_mla_dims(vllm_config.model_config).v_head_dim
|
||||
if output_dtype is None:
|
||||
output_dtype = vllm_config.model_config.dtype
|
||||
if dcp_world_size <= 1 or max_tokens <= 0 or num_heads <= 0:
|
||||
return []
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
output_dtype=output_dtype,
|
||||
lse_dtype=torch.float32,
|
||||
num_tokens=WarmupIntRange(1, max_tokens + 1),
|
||||
num_heads=num_heads,
|
||||
head_dim=head_dim,
|
||||
n_rounded=dcp_world_size,
|
||||
lse_idx=WarmupIntRange(0, dcp_world_size),
|
||||
is_base_e=is_base_e,
|
||||
)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
output_ptr = TritonWarmupTensor(
|
||||
compile_key.output_dtype,
|
||||
shape=(1, 1, compile_key.head_dim),
|
||||
strides=(
|
||||
compile_key.outputs_stride_b,
|
||||
compile_key.outputs_stride_h,
|
||||
compile_key.outputs_stride_d,
|
||||
),
|
||||
)
|
||||
lse_ptr = TritonWarmupTensor(
|
||||
compile_key.lse_dtype,
|
||||
shape=(compile_key.n_rounded, 1, 1),
|
||||
strides=(
|
||||
compile_key.lses_stride_n,
|
||||
compile_key.lses_stride_b,
|
||||
compile_key.lses_stride_h,
|
||||
),
|
||||
)
|
||||
return dict(
|
||||
outputs=output_ptr,
|
||||
new_output=output_ptr,
|
||||
lses=lse_ptr,
|
||||
vlse=lse_ptr,
|
||||
lse_idx=compile_key.lse_idx,
|
||||
ctx=None,
|
||||
is_base_e=compile_key.is_base_e,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
outputs: torch.Tensor,
|
||||
new_output: torch.Tensor,
|
||||
lses: torch.Tensor,
|
||||
vlse: torch.Tensor,
|
||||
lse_idx: int,
|
||||
ctx: Any,
|
||||
*,
|
||||
is_base_e: bool,
|
||||
) -> LaunchSpec:
|
||||
num_tokens, num_heads, head_dim = outputs.shape
|
||||
n_rounded = lses.shape[0]
|
||||
outputs_stride_b, outputs_stride_h, outputs_stride_d = outputs.stride()
|
||||
lses_stride_n, lses_stride_b, lses_stride_h = lses.stride()
|
||||
grid = (num_tokens, num_heads, 1)
|
||||
return grid, dict(
|
||||
outputs_stride_B=outputs_stride_b,
|
||||
outputs_stride_H=outputs_stride_h,
|
||||
outputs_stride_D=outputs_stride_d,
|
||||
lses_stride_N=lses_stride_n,
|
||||
lses_stride_B=lses_stride_b,
|
||||
lses_stride_H=lses_stride_h,
|
||||
HEAD_DIM=head_dim,
|
||||
N_ROUNDED=n_rounded,
|
||||
IS_BASE_E=is_base_e,
|
||||
_runtime_launcher=None if self._warming else ctx.call_kernel,
|
||||
# CPTritonContext caches the non-constexpr positional prefix; derive
|
||||
# its length so adding/reordering a kernel arg cannot silently
|
||||
# misalign the cached replay path.
|
||||
_runtime_launcher_arg_count=len(self.kernel.arg_names)
|
||||
- len(self.kernel.constexprs),
|
||||
)
|
||||
|
||||
|
||||
class CPTritonContext:
|
||||
@@ -202,14 +374,12 @@ def correct_attn_out(
|
||||
f"got {tuple(lses.shape)}"
|
||||
)
|
||||
|
||||
B, H, D = out.shape
|
||||
N = lses.shape[0]
|
||||
B, H, _ = out.shape
|
||||
|
||||
# Strides after we normalized shapes to 3-D views. The kernel computes
|
||||
# offsets for `vlse_ptr` using lses_stride_B/H, so the output buffer must
|
||||
# have the same B/H stride layout as a slice of `lses`.
|
||||
o_sB, o_sH, o_sD = out.stride()
|
||||
l_sN, l_sB, l_sH = lses.stride()
|
||||
_, l_sB, l_sH = lses.stride()
|
||||
|
||||
# Allocate LSE with the same B/H strides as `lses` so writes land correctly
|
||||
# even when `lses` is a non-contiguous view (e.g., 4-D to 3-D squeeze).
|
||||
@@ -217,24 +387,15 @@ def correct_attn_out(
|
||||
(B, H), (l_sB, l_sH), device=lses.device, dtype=lses.dtype
|
||||
)
|
||||
|
||||
# Kernel launch config
|
||||
grid = (B, H, 1)
|
||||
|
||||
regular_args = (
|
||||
_CORRECT_ATTN_CP_OUT_KERNEL(
|
||||
out,
|
||||
out,
|
||||
lses,
|
||||
lse,
|
||||
o_sB,
|
||||
o_sH,
|
||||
o_sD,
|
||||
l_sN,
|
||||
l_sB,
|
||||
l_sH,
|
||||
cp_rank,
|
||||
ctx,
|
||||
is_base_e=is_lse_base_on_e,
|
||||
)
|
||||
const_args = {"HEAD_DIM": D, "N_ROUNDED": N, "IS_BASE_E": is_lse_base_on_e}
|
||||
ctx.call_kernel(_correct_attn_cp_out_kernel, grid, *regular_args, **const_args)
|
||||
return out, lse
|
||||
|
||||
|
||||
@@ -1250,6 +1411,14 @@ class MLADCPManager:
|
||||
is_lse_base_on_e,
|
||||
use_pcp,
|
||||
)
|
||||
if vllm_config.kernel_config.enable_jit_warmup and not self.use_a2a:
|
||||
_CORRECT_ATTN_CP_OUT_KERNEL.register_warmup(
|
||||
vllm_config,
|
||||
output_dtype=output_dtype,
|
||||
num_heads=num_heads,
|
||||
head_dim=output_head_dim,
|
||||
is_base_e=is_lse_base_on_e,
|
||||
)
|
||||
self.query_gather = (
|
||||
None
|
||||
if use_pcp
|
||||
@@ -1411,3 +1580,6 @@ class MLADCPManager:
|
||||
local_kv: torch.Tensor,
|
||||
) -> object:
|
||||
return self._kv_gather(gathered_kv, local_kv)
|
||||
|
||||
|
||||
_CORRECT_ATTN_CP_OUT_KERNEL = CorrectAttnCPOutKernel()
|
||||
|
||||
@@ -11,11 +11,11 @@ import torch
|
||||
|
||||
from vllm.distributed import get_dcp_group, get_pcp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.warmup.jit_warmup import (
|
||||
VllmJitKernel,
|
||||
)
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
kernel_launcher,
|
||||
triton_scalar_specialization_rep,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
@@ -394,7 +394,9 @@ class MultiGroupBlockTable:
|
||||
return self.block_tables[idx]
|
||||
|
||||
|
||||
class ComputeSlotMappingKernel(VllmJitKernel["ComputeSlotMappingKernel.CompileKey"]):
|
||||
class ComputeSlotMappingKernel(
|
||||
VllmTritonJitKernel["ComputeSlotMappingKernel.CompileKey"]
|
||||
):
|
||||
triton_block_size = 1024
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -490,37 +492,53 @@ class ComputeSlotMappingKernel(VllmJitKernel["ComputeSlotMappingKernel.CompileKe
|
||||
def get_warmup_keys(self, **dispatch_kwargs: int) -> list[CompileKey]:
|
||||
return self._trace_dispatch(self.dispatch)(**dispatch_kwargs)
|
||||
|
||||
def compile(self, compile_key: CompileKey) -> None:
|
||||
warmup = getattr(self.kernel, "warmup", None)
|
||||
assert warmup is not None
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
int32_ptr = TritonWarmupTensor(torch.int32)
|
||||
int64_ptr = TritonWarmupTensor(torch.int64)
|
||||
warmup(
|
||||
2, # arbitrary, num_tokens in do_not_specialize
|
||||
2, # arbitrary, max_num_tokens in do_not_specialize
|
||||
int32_ptr,
|
||||
int64_ptr,
|
||||
int32_ptr,
|
||||
compile_key.block_table_stride,
|
||||
compile_key.block_size,
|
||||
int64_ptr,
|
||||
KV_CACHE_BLOCK_SIZE=compile_key.kv_cache_block_size,
|
||||
BLOCKS_PER_KV_BLOCK=compile_key.blocks_per_kv_block,
|
||||
TOTAL_CP_WORLD_SIZE=compile_key.total_cp_world_size,
|
||||
TOTAL_CP_RANK=compile_key.total_cp_rank,
|
||||
CP_KV_CACHE_INTERLEAVE_SIZE=compile_key.cp_kv_cache_interleave_size,
|
||||
PAD_ID=PAD_SLOT_ID,
|
||||
BLOCK_SIZE=self.triton_block_size,
|
||||
grid=(2,),
|
||||
return dict(
|
||||
num_reqs=1,
|
||||
num_tokens=2, # arbitrary, in do_not_specialize
|
||||
max_num_tokens=2, # arbitrary, in do_not_specialize
|
||||
query_start_loc=int32_ptr,
|
||||
positions=int64_ptr,
|
||||
block_table=TritonWarmupTensor(
|
||||
torch.int32,
|
||||
shape=(1, compile_key.block_table_stride),
|
||||
),
|
||||
block_table_stride=compile_key.block_table_stride,
|
||||
block_size=compile_key.block_size,
|
||||
slot_mapping=int64_ptr,
|
||||
kv_cache_block_size=compile_key.kv_cache_block_size,
|
||||
blocks_per_kv_block=compile_key.blocks_per_kv_block,
|
||||
total_cp_world_size=compile_key.total_cp_world_size,
|
||||
total_cp_rank=compile_key.total_cp_rank,
|
||||
cp_kv_cache_interleave_size=compile_key.cp_kv_cache_interleave_size,
|
||||
)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
num_reqs: int,
|
||||
*args: Any,
|
||||
) -> None:
|
||||
self.kernel[(num_reqs + 1,)](
|
||||
*args,
|
||||
num_tokens: int,
|
||||
max_num_tokens: int,
|
||||
query_start_loc: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
block_table_stride: int,
|
||||
block_size: int,
|
||||
slot_mapping: torch.Tensor,
|
||||
kv_cache_block_size: int,
|
||||
blocks_per_kv_block: int,
|
||||
total_cp_world_size: int,
|
||||
total_cp_rank: int,
|
||||
cp_kv_cache_interleave_size: int,
|
||||
) -> LaunchSpec:
|
||||
return (num_reqs + 1,), dict(
|
||||
KV_CACHE_BLOCK_SIZE=kv_cache_block_size,
|
||||
BLOCKS_PER_KV_BLOCK=blocks_per_kv_block,
|
||||
TOTAL_CP_WORLD_SIZE=total_cp_world_size,
|
||||
TOTAL_CP_RANK=total_cp_rank,
|
||||
CP_KV_CACHE_INTERLEAVE_SIZE=cp_kv_cache_interleave_size,
|
||||
PAD_ID=PAD_SLOT_ID,
|
||||
BLOCK_SIZE=self.triton_block_size,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user