[2/N][warmup][DSv4] Migrate sequence and DCP kernels (#53564)

Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Roberto L. Castro
2026-09-07 09:35:54 -04:00
committed by GitHub
co-authored by Codex OpenAI Codex Lucas Wilkinson mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ecd600d91e
commit 1713b9866a
12 changed files with 1440 additions and 562 deletions
@@ -13,6 +13,8 @@ steps:
- vllm/model_executor/warmup
- tests/model_executor
- tests/model_executor/test_jit_warmup.py
- tests/model_executor/test_jit_warmup_cutedsl_launcher.py
- tests/model_executor/test_jit_warmup_triton_launcher.py
- tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
commands:
- apt-get update && apt-get install -y curl libsodium23
@@ -41,6 +43,8 @@ steps:
- vllm/model_executor/warmup
- tests/model_executor
- tests/model_executor/test_jit_warmup.py
- tests/model_executor/test_jit_warmup_cutedsl_launcher.py
- tests/model_executor/test_jit_warmup_triton_launcher.py
- tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
+3 -1
View File
@@ -532,10 +532,12 @@ def test_runtime_cache_miss_compiles_and_caches_executor() -> None:
def test_registry_records_only_inside_model_setup_context() -> None:
registry = JitWarmupRegistry(_config())
kernel = RecordingToyKernel()
config = _config()
kernel.register_warmup(3, _config())
with registry.activate():
kernel.register_warmup(3, _config())
kernel.register_warmup(3, config)
kernel.register_warmup(3, config)
assert len(registry) == 1
assert kernel.compiled == []
@@ -0,0 +1,76 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any
from vllm.model_executor.warmup import jit_warmup_cutedsl_helper
from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
CuTeDSLLaunchSpec,
VllmCuTeDSLJitKernel,
cutedsl_kernel_launcher,
)
class _TestCuTeDSLKernel(VllmCuTeDSLJitKernel["_TestCuTeDSLKernel.CompileKey"]):
@dataclass(frozen=True)
class CompileKey:
variant: int
@staticmethod
def kernel(compile_key: CompileKey) -> tuple[str, int]:
return "entry", compile_key.variant
def dispatch(self, *, variant: int) -> CompileKey:
return self.CompileKey(variant=variant)
def get_warmup_keys(self) -> list[CompileKey]:
return self._trace_dispatch(self.dispatch)(variant=(1, 2))
def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
return (f"fake-{compile_key.variant}",)
@cutedsl_kernel_launcher
def __call__(
self,
payload: str,
*,
variant: int,
) -> CuTeDSLLaunchSpec[CompileKey]:
compile_key = self.dispatch(variant=variant)
return compile_key, (payload,), {"variant": variant}
def test_cutedsl_launcher_reuses_compiled_executor(monkeypatch) -> None:
compile_calls: list[tuple[Any, tuple[Any, ...]]] = []
launch_calls: list[tuple[int, tuple[Any, ...]]] = []
def compile_cutedsl(entry: tuple[str, int], *args: Any) -> Any:
compile_calls.append((entry, args))
variant = entry[1]
def executor(*runtime_args: Any) -> tuple[int, tuple[Any, ...]]:
launch_calls.append((variant, runtime_args))
return variant, runtime_args
return executor
monkeypatch.setattr(
jit_warmup_cutedsl_helper,
"compile_cutedsl",
compile_cutedsl,
)
kernel = _TestCuTeDSLKernel()
compile_key = kernel.CompileKey(variant=1)
kernel.compile(compile_key)
kernel.compile(compile_key)
assert compile_calls == [(("entry", 1), ("fake-1",))]
assert kernel("runtime", variant=1) == (1, ("runtime",))
assert kernel("other", variant=2) == (2, ("other",))
assert compile_calls == [
(("entry", 1), ("fake-1",)),
(("entry", 2), ("fake-2",)),
]
assert launch_calls == [(1, ("runtime",)), (2, ("other",))]
@@ -0,0 +1,128 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
VllmTritonJitKernel,
kernel_launcher,
)
class _FakeTritonKernel:
arg_names = ("first", "second", "CONST")
def __init__(self) -> None:
self.warmup_calls: list[dict[str, Any]] = []
def warmup(self, **kwargs: Any) -> None:
self.warmup_calls.append(kwargs)
class _TestTritonKernel(VllmTritonJitKernel["_TestTritonKernel.CompileKey"]):
kernel = _FakeTritonKernel()
@dataclass(frozen=True)
class CompileKey:
value: int
def dispatch(self, *, value: int) -> CompileKey:
return self.CompileKey(value=value)
def get_warmup_keys(self) -> list[CompileKey]:
return self._trace_dispatch(self.dispatch)(value=1)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
return dict(first="warmup", second=compile_key.value, runtime_launcher=None)
@kernel_launcher
def __call__(
self,
first: str,
second: int,
runtime_launcher: Any,
) -> LaunchSpec:
return (2,), dict(
CONST=7,
_runtime_launcher=runtime_launcher,
_runtime_launcher_arg_count=2,
)
def test_triton_launcher_supports_compile_and_runtime_adapters() -> None:
owner = _TestTritonKernel()
owner.kernel.warmup_calls.clear()
owner.compile(owner.CompileKey(value=1))
assert owner.kernel.warmup_calls == [
{"grid": (1,), "first": "warmup", "second": 1, "CONST": 7}
]
runtime_calls: list[tuple[Any, ...]] = []
def runtime_launcher(kernel: Any, grid: Any, *args: Any, **kwargs: Any) -> None:
runtime_calls.append((kernel, grid, args, kwargs))
owner("runtime", 2, runtime_launcher)
assert runtime_calls == [(owner.kernel, (2,), ("runtime", 2), {"CONST": 7})]
def test_triton_launcher_supports_cpu_function_wrappers() -> None:
calls: list[tuple[Any, ...]] = []
def kernel(first: str, second: int, CONST: int) -> None:
calls.append((first, second, CONST))
class FuncWrapper:
def __init__(self) -> None:
self.func = kernel
def __getitem__(self, _grid: Any) -> Any:
return self.func
class TestCpuKernel(_TestTritonKernel):
kernel: Any = FuncWrapper()
owner = TestCpuKernel()
owner("runtime", 2, None)
assert calls == [("runtime", 2, 7)]
def test_compute_slot_mapping_uses_named_launcher_inputs(monkeypatch) -> None:
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.block_table import ComputeSlotMappingKernel
owner = ComputeSlotMappingKernel()
compile_key = owner.CompileKey(
kv_cache_block_size=16,
blocks_per_kv_block=1,
total_cp_world_size=2,
total_cp_rank=1,
cp_kv_cache_interleave_size=1,
block_table_stride=128,
block_size=16,
)
launches: list[tuple[Any, ...]] = []
def launch(grid: Any, inputs: Any, **kwargs: Any) -> None:
launches.append((grid, inputs, kwargs))
monkeypatch.setattr(owner, "launch", launch)
owner.compile(compile_key)
grid, inputs, kwargs = launches[0]
assert grid == (2,)
assert inputs["num_tokens"] == 2
assert inputs["block_table_stride"] == 128
assert kwargs == {
"KV_CACHE_BLOCK_SIZE": 16,
"BLOCKS_PER_KV_BLOCK": 1,
"TOTAL_CP_WORLD_SIZE": 2,
"TOTAL_CP_RANK": 1,
"CP_KV_CACHE_INTERLEAVE_SIZE": 1,
"PAD_ID": PAD_SLOT_ID,
"BLOCK_SIZE": owner.triton_block_size,
}
@@ -1,7 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import cache
from dataclasses import dataclass
from typing import Any
import cutlass
import cutlass.cute as cute
@@ -11,6 +12,17 @@ from cutlass import Float32, Int32, Uint32, Uint64
from quack.compile_utils import make_fake_tensor
from vllm.cute_utils import recast_val
from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
CuTeDSLLaunchSpec,
VllmCuTeDSLJitKernel,
cutedsl_kernel_launcher,
)
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
kernel_launcher,
)
from vllm.triton_utils import tl, triton
@@ -25,9 +37,7 @@ def stable_topk_from_gathered_candidates_cutedsl(
dtype=torch.int32,
device=gathered.device,
)
StableTopKFromGatheredCandidatesKernel.compile(topk, gathered.shape[1])(
gathered, out
)
_STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL(gathered, out, topk=topk)
return out
@@ -41,380 +51,543 @@ def pack_dcp_topk_candidates_cutedsl(
row_starts: torch.Tensor | None,
) -> None:
topk = topk_indices.shape[1]
grid = (topk_indices.shape[0], triton.cdiv(topk, 512))
row_starts_arg = row_starts if row_starts is not None else topk_indices
_pack_dcp_topk_candidates_triton_kernel[grid](
_PACK_DCP_TOPK_CANDIDATES_KERNEL(
logits,
topk_indices,
packed,
row_starts_arg,
logits.stride(0),
logits.stride(1),
topk_indices.stride(0),
topk_indices.stride(1),
packed.stride(0),
packed.stride(1),
packed.stride(2),
logits.shape[1],
DCP_RANK=dcp_rank,
DCP_WORLD_SIZE=dcp_world_size,
CP_INTERLEAVE=cp_interleave,
HAS_ROW_STARTS=row_starts is not None,
TOPK=topk,
BLOCK_SIZE=512,
num_warps=8,
logits_stride0=logits.stride(0),
logits_stride1=logits.stride(1),
topk_stride0=topk_indices.stride(0),
topk_stride1=topk_indices.stride(1),
packed_stride0=packed.stride(0),
packed_stride1=packed.stride(1),
packed_stride2=packed.stride(2),
num_cols=logits.shape[1],
dcp_rank=dcp_rank,
dcp_world_size=dcp_world_size,
cp_interleave=cp_interleave,
has_row_starts=row_starts is not None,
topk=topk,
block_size=512,
)
@triton.jit
def _pack_dcp_topk_candidates_triton_kernel(
logits,
topk_indices,
packed,
row_starts,
logits_stride0: tl.constexpr,
logits_stride1: tl.constexpr,
topk_stride0: tl.constexpr,
topk_stride1: tl.constexpr,
packed_stride0: tl.constexpr,
packed_stride1: tl.constexpr,
packed_stride2: tl.constexpr,
num_cols,
DCP_RANK: tl.constexpr,
DCP_WORLD_SIZE: tl.constexpr,
CP_INTERLEAVE: tl.constexpr,
HAS_ROW_STARTS: tl.constexpr,
TOPK: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
class PackDCPTopkCandidatesKernel(
VllmTritonJitKernel["PackDCPTopkCandidatesKernel.CompileKey"]
):
row = tl.program_id(0)
tile = tl.program_id(1)
cols = tile * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = cols < TOPK
@dataclass(frozen=True)
class CompileKey:
dcp_rank: int
dcp_world_size: int
cp_interleave: int
has_row_starts: bool
topk: int
block_size: int
local_idx = tl.load(
topk_indices + row * topk_stride0 + cols * topk_stride1,
mask=mask,
other=-1,
# These scalars only describe runtime layouts and bounds; the constexpr
# fields below own the launch geometry and algorithmic specialization.
@staticmethod
@triton.jit(
do_not_specialize=[
"logits_stride0",
"logits_stride1",
"topk_stride0",
"topk_stride1",
"packed_stride0",
"packed_stride1",
"packed_stride2",
"num_cols",
]
)
valid = local_idx >= 0
safe_local_idx = tl.maximum(local_idx, 0)
def kernel(
logits,
topk_indices,
packed,
row_starts,
logits_stride0,
logits_stride1,
topk_stride0,
topk_stride1,
packed_stride0,
packed_stride1,
packed_stride2,
num_cols,
DCP_RANK: tl.constexpr,
DCP_WORLD_SIZE: tl.constexpr,
CP_INTERLEAVE: tl.constexpr,
HAS_ROW_STARTS: tl.constexpr,
TOPK: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
row = tl.program_id(0)
tile = tl.program_id(1)
cols = tile * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = cols < TOPK
row_start = 0
if HAS_ROW_STARTS:
row_start = tl.load(row_starts + row)
local_idx = tl.load(
topk_indices + row * topk_stride0 + cols * topk_stride1,
mask=mask,
other=-1,
)
valid = local_idx >= 0
safe_local_idx = tl.maximum(local_idx, 0)
score_col = safe_local_idx + row_start
score_col = tl.minimum(score_col, tl.maximum(num_cols - 1, 0))
score = tl.load(
logits + row * logits_stride0 + score_col * logits_stride1,
mask=mask & valid,
other=-float("inf"),
)
row_start = 0
if HAS_ROW_STARTS:
row_start = tl.load(row_starts + row)
global_id = (
(safe_local_idx // CP_INTERLEAVE) * (DCP_WORLD_SIZE * CP_INTERLEAVE)
+ DCP_RANK * CP_INTERLEAVE
+ safe_local_idx % CP_INTERLEAVE
)
global_id = tl.where(valid, global_id, -1)
score_col = safe_local_idx + row_start
score_col = tl.minimum(score_col, tl.maximum(num_cols - 1, 0))
score = tl.load(
logits + row * logits_stride0 + score_col * logits_stride1,
mask=mask & valid,
other=-float("inf"),
)
packed_base = packed + row * packed_stride0 + cols * packed_stride1
tl.store(packed_base, score, mask=mask)
tl.store(packed_base + packed_stride2, global_id.to(tl.float32), mask=mask)
global_id = (
(safe_local_idx // CP_INTERLEAVE) * (DCP_WORLD_SIZE * CP_INTERLEAVE)
+ DCP_RANK * CP_INTERLEAVE
+ safe_local_idx % CP_INTERLEAVE
)
global_id = tl.where(valid, global_id, -1)
packed_base = packed + row * packed_stride0 + cols * packed_stride1
tl.store(packed_base, score, mask=mask)
tl.store(packed_base + packed_stride2, global_id.to(tl.float32), mask=mask)
def dispatch( # type: ignore[override]
self,
*,
has_row_starts: bool,
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(
**compile_key_fields,
has_row_starts=has_row_starts,
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
if dcp_world_size <= 1:
return []
cp_interleave = vllm_config.parallel_config.cp_kv_cache_interleave_size
topk = vllm_config.model_config.hf_config.index_topk
if topk <= 0:
return []
try:
from vllm.distributed.parallel_state import get_dcp_group
dcp_rank = get_dcp_group().rank_in_group
except Exception:
dcp_rank = 0
return self._trace_dispatch(self.dispatch)(
dcp_rank=dcp_rank,
dcp_world_size=dcp_world_size,
cp_interleave=cp_interleave,
has_row_starts=(False, True),
topk=topk,
block_size=512,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
fp32_ptr = TritonWarmupTensor(torch.float32)
int32_ptr = TritonWarmupTensor(torch.int32)
return dict(
logits=fp32_ptr,
topk_indices=TritonWarmupTensor(torch.int32, shape=(1, compile_key.topk)),
packed=fp32_ptr,
row_starts_arg=int32_ptr,
logits_stride0=1,
logits_stride1=1,
topk_stride0=1,
topk_stride1=1,
packed_stride0=1,
packed_stride1=1,
packed_stride2=1,
num_cols=1,
dcp_rank=compile_key.dcp_rank,
dcp_world_size=compile_key.dcp_world_size,
cp_interleave=compile_key.cp_interleave,
has_row_starts=compile_key.has_row_starts,
topk=compile_key.topk,
block_size=compile_key.block_size,
)
@kernel_launcher
def __call__(
self,
logits: torch.Tensor,
topk_indices: torch.Tensor,
packed: torch.Tensor,
row_starts_arg: torch.Tensor,
*,
logits_stride0: int,
logits_stride1: int,
topk_stride0: int,
topk_stride1: int,
packed_stride0: int,
packed_stride1: int,
packed_stride2: int,
num_cols: int,
dcp_rank: int,
dcp_world_size: int,
cp_interleave: int,
has_row_starts: bool,
topk: int,
block_size: int,
) -> LaunchSpec:
grid = (topk_indices.shape[0], triton.cdiv(topk, block_size))
return grid, dict(
row_starts=row_starts_arg,
DCP_RANK=dcp_rank,
DCP_WORLD_SIZE=dcp_world_size,
CP_INTERLEAVE=cp_interleave,
HAS_ROW_STARTS=has_row_starts,
TOPK=topk,
BLOCK_SIZE=block_size,
num_warps=8,
)
@cute.jit
def _warp_scan_inclusive_i32(val: Int32, lane: Int32) -> Int32:
for i in cutlass.range_constexpr(cute.arch.WARP_SIZE.bit_length() - 1):
offset = 1 << i
partial = cute.arch.shuffle_sync_up(val, offset=offset, mask_and_clamp=0)
if lane >= offset:
val += partial
return val
@cute.jit
def _block_scan_inclusive_i32(
val: Int32,
lane: Int32,
warp_id: Int32,
warp_scratch: cute.Tensor,
warps_per_block: int,
) -> Int32:
prefix = _warp_scan_inclusive_i32(val, lane)
if lane == Int32(cute.arch.WARP_SIZE - 1):
warp_scratch[0, warp_id] = prefix
cute.arch.sync_threads()
if warp_id == Int32(0):
warp_total = Int32(0)
if lane < Int32(warps_per_block):
warp_total = warp_scratch[0, lane]
warp_prefix = _warp_scan_inclusive_i32(warp_total, lane)
if lane < Int32(warps_per_block):
warp_scratch[0, lane] = warp_prefix - warp_total
cute.arch.sync_threads()
return prefix + warp_scratch[0, warp_id]
class StableTopKFromGatheredCandidatesKernel:
class StableTopKFromGatheredCandidatesKernel(
VllmCuTeDSLJitKernel["StableTopKFromGatheredCandidatesKernel.CompileKey"]
):
tb_size = 512
hist_bins = 2048
radix_bits = (hist_bins - 1).bit_length()
assert hist_bins == 1 << radix_bits
key_bits = Uint64.width
radix_passes = (key_bits + radix_bits - 1) // radix_bits
final_radix_bits = key_bits - radix_bits * (radix_passes - 1)
hist_chunks = (hist_bins + tb_size - 1) // tb_size
warps_per_block = tb_size // cute.arch.WARP_SIZE
def __init__(self, topk: int, num_candidates: int):
assert num_candidates % self.tb_size == 0, (
@dataclass(frozen=True)
class CompileKey:
topk: int
num_candidates: int
@staticmethod
def kernel(compile_key: CompileKey) -> Any:
tb_size = StableTopKFromGatheredCandidatesKernel.tb_size
hist_bins = StableTopKFromGatheredCandidatesKernel.hist_bins
radix_bits = StableTopKFromGatheredCandidatesKernel.radix_bits
key_bits = StableTopKFromGatheredCandidatesKernel.key_bits
radix_passes = StableTopKFromGatheredCandidatesKernel.radix_passes
final_radix_bits = StableTopKFromGatheredCandidatesKernel.final_radix_bits
hist_chunks = StableTopKFromGatheredCandidatesKernel.hist_chunks
warps_per_block = StableTopKFromGatheredCandidatesKernel.warps_per_block
topk = compile_key.topk
assert hist_bins == 1 << radix_bits
assert compile_key.num_candidates % tb_size == 0, (
"StableTopKFromGatheredCandidatesKernel requires candidate count "
f"to be a multiple of {self.tb_size}, got {num_candidates}"
f"to be a multiple of {tb_size}, got {compile_key.num_candidates}"
)
self.topk = topk
self.keys_per_thread = num_candidates // self.tb_size
keys_per_thread = compile_key.num_candidates // tb_size
@cute.struct
class SharedStorage:
hist: cute.struct.MemRange[Int32, self.hist_bins]
hist: cute.struct.MemRange[Int32, hist_bins]
committed_count: cute.struct.MemRange[Int32, 1]
running_count: cute.struct.MemRange[Int32, 1]
threshold_bin: cute.struct.MemRange[Int32, 1]
threshold_found: cute.struct.MemRange[Int32, 1]
include_threshold_bin: cute.struct.MemRange[Int32, 1]
prefix_s: cute.struct.Align[cute.struct.MemRange[Uint64, 1], 8]
warp_totals: cute.struct.MemRange[Int32, self.warps_per_block]
warp_totals: cute.struct.MemRange[Int32, warps_per_block]
self.shared_storage = SharedStorage
shared_storage = SharedStorage
@cute.jit
def __call__(
self,
gathered: cute.Tensor,
out: cute.Tensor,
stream: CUstream,
):
grid = (gathered.shape[0], 1, 1)
self.kernel(gathered, out).launch(
grid=grid,
block=(self.tb_size, 1, 1),
stream=stream,
)
@cute.jit
def _stable_key(self, score: Float32, token_id: Int32) -> Uint64:
bits = recast_val(score, Uint32)
mask = Uint32(0x80000000)
if (bits & Uint32(0x80000000)) != Uint32(0):
mask = Uint32(0xFFFFFFFF)
score_key = Uint64(bits ^ mask) << Uint64(32)
id_key = Uint64(~Uint32(token_id))
key = score_key | id_key
if token_id < Int32(0):
key = Uint64(0)
return key
@cute.jit
def _prefix_matches(
self,
key: Uint64,
prefix: Uint64,
prefix_bits: Int32,
):
matches = prefix_bits == Int32(0)
if prefix_bits != Int32(0):
shift = Int32(self.key_bits) - prefix_bits
matches = (key >> Uint64(shift)) == (prefix >> Uint64(shift))
return matches
@cute.jit
def _radix_pass(
self,
keys: cute.Tensor,
output: cute.Tensor,
storage,
tid: Int32,
step: Int32,
bits: int,
is_final_pass: bool,
):
hist_smem = storage.hist.get_tensor(cute.make_layout((self.hist_bins,)))
committed_count_smem = storage.committed_count.data_ptr()
running_count_smem = storage.running_count.data_ptr()
threshold_bin_smem = storage.threshold_bin.data_ptr()
threshold_found_smem = storage.threshold_found.data_ptr()
include_threshold_bin_smem = storage.include_threshold_bin.data_ptr()
prefix_smem = storage.prefix_s.data_ptr()
warp_totals_smem = storage.warp_totals.get_tensor(
cute.make_layout((1, self.warps_per_block))
)
prefix_bits = step * Int32(self.radix_bits)
num_bins = 1 << bits
block_scan_iterations = (num_bins + self.tb_size - 1) // self.tb_size
shift = Int32(self.key_bits) - prefix_bits - Int32(bits)
bin_mask = Uint64(num_bins - 1)
prefix = prefix_smem.load()
for chunk in cutlass.range_constexpr(self.hist_chunks):
hist_smem[tid + Int32(chunk * self.tb_size)] = Int32(0)
if tid == Int32(0):
running_count_smem.store(committed_count_smem.load())
include_threshold_bin_smem.store(Int32(0))
threshold_found_smem.store(Int32(0))
cute.arch.sync_threads()
for key_idx in cutlass.range_constexpr(self.keys_per_thread):
key = keys[key_idx]
if self._prefix_matches(key, prefix, prefix_bits):
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
cute.arch.atomic_add(
hist_smem.iterator + bin_idx,
Int32(1),
sem="relaxed",
scope="cta",
@cute.jit
def warp_scan_inclusive_i32(val: Int32, lane: Int32) -> Int32:
for i in cutlass.range_constexpr(cute.arch.WARP_SIZE.bit_length() - 1):
offset = 1 << i
partial = cute.arch.shuffle_sync_up(
val, offset=offset, mask_and_clamp=0
)
cute.arch.sync_threads()
if lane >= offset:
val += partial
return val
lane = cute.arch.lane_idx()
warp_id = cute.arch.warp_idx()
# Each iteration scans one tb_size-wide slice of bins, high to low.
iter = Int32(0)
threshold_found = threshold_found_smem.load()
while threshold_found == Int32(0) and iter < Int32(block_scan_iterations):
bin_idx = Int32(num_bins - 1) - (iter * Int32(self.tb_size) + tid)
count = hist_smem[bin_idx]
chunk_inclusive = _block_scan_inclusive_i32(
count,
lane,
warp_id,
warp_totals_smem,
self.warps_per_block,
@cute.jit
def block_scan_inclusive_i32(
val: Int32,
lane: Int32,
warp_id: Int32,
warp_scratch: cute.Tensor,
warps_per_block: int,
) -> Int32:
prefix = warp_scan_inclusive_i32(val, lane)
if lane == Int32(cute.arch.WARP_SIZE - 1):
warp_scratch[0, warp_id] = prefix
cute.arch.sync_threads()
if warp_id == Int32(0):
warp_total = Int32(0)
if lane < Int32(warps_per_block):
warp_total = warp_scratch[0, lane]
warp_prefix = warp_scan_inclusive_i32(warp_total, lane)
if lane < Int32(warps_per_block):
warp_scratch[0, lane] = warp_prefix - warp_total
cute.arch.sync_threads()
return prefix + warp_scratch[0, warp_id]
@cute.jit
def stable_key(score: Float32, token_id: Int32) -> Uint64:
bits = recast_val(score, Uint32)
mask = Uint32(0x80000000)
if (bits & Uint32(0x80000000)) != Uint32(0):
mask = Uint32(0xFFFFFFFF)
score_key = Uint64(bits ^ mask) << Uint64(32)
id_key = Uint64(~Uint32(token_id))
key = score_key | id_key
if token_id < Int32(0):
key = Uint64(0)
return key
@cute.jit
def prefix_matches(
key: Uint64,
prefix: Uint64,
prefix_bits: Int32,
):
matches = prefix_bits == Int32(0)
if prefix_bits != Int32(0):
shift = Int32(key_bits) - prefix_bits
matches = (key >> Uint64(shift)) == (prefix >> Uint64(shift))
return matches
@cute.jit
def radix_pass(
keys: cute.Tensor,
output: cute.Tensor,
storage,
tid: Int32,
step: Int32,
bits: int,
is_final_pass: bool,
):
hist_smem = storage.hist.get_tensor(cute.make_layout((hist_bins,)))
committed_count_smem = storage.committed_count.data_ptr()
running_count_smem = storage.running_count.data_ptr()
threshold_bin_smem = storage.threshold_bin.data_ptr()
threshold_found_smem = storage.threshold_found.data_ptr()
include_threshold_bin_smem = storage.include_threshold_bin.data_ptr()
prefix_smem = storage.prefix_s.data_ptr()
warp_totals_smem = storage.warp_totals.get_tensor(
cute.make_layout((1, warps_per_block))
)
running_count = running_count_smem.load()
prior_in_scan_slice = chunk_inclusive - count
remaining = Int32(self.topk) - running_count - prior_in_scan_slice
if count > Int32(0) and remaining > Int32(0) and remaining <= count:
threshold_bin_smem.store(bin_idx)
if count <= remaining or cutlass.const_expr(is_final_pass):
include_threshold_bin_smem.store(Int32(1))
threshold_found_smem.store(Int32(1))
# Barrier: every thread must finish reading running_count for this
# slice before tb_size-1 advances it, else a warp racing ahead to
# the store makes a lagging thread double-count the slice total
# (-> remaining too small -> threshold too high -> under-fill).
cute.arch.sync_threads()
if tid == Int32(self.tb_size - 1):
running_count_smem.store(running_count + chunk_inclusive)
prefix_bits = step * Int32(radix_bits)
num_bins = 1 << bits
block_scan_iterations = (num_bins + tb_size - 1) // tb_size
shift = Int32(key_bits) - prefix_bits - Int32(bits)
bin_mask = Uint64(num_bins - 1)
prefix = prefix_smem.load()
for chunk in cutlass.range_constexpr(hist_chunks):
hist_smem[tid + Int32(chunk * tb_size)] = Int32(0)
if tid == Int32(0):
running_count_smem.store(committed_count_smem.load())
include_threshold_bin_smem.store(Int32(0))
threshold_found_smem.store(Int32(0))
cute.arch.sync_threads()
threshold_found = threshold_found_smem.load()
iter += Int32(1)
threshold = threshold_bin_smem.load()
should_include_threshold = include_threshold_bin_smem.load() != Int32(0)
for key_idx in cutlass.range_constexpr(self.keys_per_thread):
key = keys[key_idx]
if self._prefix_matches(key, prefix, prefix_bits):
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
selected = bin_idx > threshold
if should_include_threshold:
selected = selected or bin_idx == threshold
if selected:
dst = cute.arch.atomic_add(
committed_count_smem,
for key_idx in cutlass.range_constexpr(keys_per_thread):
key = keys[key_idx]
if prefix_matches(key, prefix, prefix_bits):
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
cute.arch.atomic_add(
hist_smem.iterator + bin_idx,
Int32(1),
sem="relaxed",
scope="cta",
)
if dst < Int32(self.topk):
output[dst] = recast_val(~Uint32(key), Int32)
cute.arch.sync_threads()
cute.arch.sync_threads()
pass_finished = include_threshold_bin_smem.load()
if tid == Int32(0) and pass_finished == Int32(0):
prefix_smem.store(prefix | (Uint64(threshold) << Uint64(shift)))
cute.arch.sync_threads()
return pass_finished
lane = cute.arch.lane_idx()
warp_id = cute.arch.warp_idx()
# Each iteration scans one tb_size-wide slice of bins, high to low.
iter = Int32(0)
threshold_found = threshold_found_smem.load()
while threshold_found == Int32(0) and iter < Int32(block_scan_iterations):
bin_idx = Int32(num_bins - 1) - (iter * Int32(tb_size) + tid)
count = hist_smem[bin_idx]
chunk_inclusive = block_scan_inclusive_i32(
count,
lane,
warp_id,
warp_totals_smem,
warps_per_block,
)
running_count = running_count_smem.load()
prior_in_scan_slice = chunk_inclusive - count
remaining = Int32(topk) - running_count - prior_in_scan_slice
if count > Int32(0) and remaining > Int32(0) and remaining <= count:
threshold_bin_smem.store(bin_idx)
if count <= remaining or cutlass.const_expr(is_final_pass):
include_threshold_bin_smem.store(Int32(1))
threshold_found_smem.store(Int32(1))
# Barrier: every thread must finish reading running_count for this
# slice before tb_size-1 advances it, else a warp racing ahead to
# the store makes a lagging thread double-count the slice total
# (-> remaining too small -> threshold too high -> under-fill).
cute.arch.sync_threads()
if tid == Int32(tb_size - 1):
running_count_smem.store(running_count + chunk_inclusive)
cute.arch.sync_threads()
@cute.kernel
def kernel(
threshold_found = threshold_found_smem.load()
iter += Int32(1)
threshold = threshold_bin_smem.load()
should_include_threshold = include_threshold_bin_smem.load() != Int32(0)
for key_idx in cutlass.range_constexpr(keys_per_thread):
key = keys[key_idx]
if prefix_matches(key, prefix, prefix_bits):
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
selected = bin_idx > threshold
if should_include_threshold:
selected = selected or bin_idx == threshold
if selected:
dst = cute.arch.atomic_add(
committed_count_smem,
Int32(1),
sem="relaxed",
scope="cta",
)
if dst < Int32(topk):
output[dst] = recast_val(~Uint32(key), Int32)
cute.arch.sync_threads()
pass_finished = include_threshold_bin_smem.load()
if tid == Int32(0) and pass_finished == Int32(0):
prefix_smem.store(prefix | (Uint64(threshold) << Uint64(shift)))
cute.arch.sync_threads()
return pass_finished
@cute.kernel
def device_kernel(input: cute.Tensor, out: cute.Tensor):
row, _, _ = cute.arch.block_idx()
tid, _, _ = cute.arch.thread_idx()
input_row = input[row, None, None]
output_row = out[row, None]
keys = cute.make_rmem_tensor((keys_per_thread,), Uint64)
smem = cutlass.utils.SmemAllocator()
storage = smem.allocate(shared_storage, 8)
committed_count_smem = storage.committed_count.data_ptr()
prefix_smem = storage.prefix_s.data_ptr()
for i in range(tid, topk, tb_size):
output_row[i] = Int32(-1)
for key_idx in cutlass.range_constexpr(keys_per_thread):
col = tid + Int32(key_idx * tb_size)
score = Float32(input_row[col, 0])
token_id = Int32(input_row[col, 1])
keys[key_idx] = stable_key(score, token_id)
if tid == Int32(0):
committed_count_smem.store(Int32(0))
prefix_smem.store(Uint64(0))
cute.arch.sync_threads()
step = Int32(0)
finished = Int32(0)
while finished == Int32(0) and step < Int32(radix_passes - 1):
finished = radix_pass(
keys,
output_row,
storage,
tid,
step,
radix_bits,
False,
)
step += Int32(1)
if finished == Int32(0):
radix_pass(
keys,
output_row,
storage,
tid,
Int32(radix_passes - 1),
final_radix_bits,
True,
)
@cute.jit
def host_entrypoint(
gathered: cute.Tensor,
out: cute.Tensor,
stream: CUstream,
):
grid = (gathered.shape[0], 1, 1)
device_kernel(gathered, out).launch(
grid=grid,
block=(tb_size, 1, 1),
stream=stream,
)
return host_entrypoint
def dispatch( # type: ignore[override]
self,
input: cute.Tensor,
out: cute.Tensor,
):
row, _, _ = cute.arch.block_idx()
tid, _, _ = cute.arch.thread_idx()
input_row = input[row, None, None]
output_row = out[row, None]
keys = cute.make_rmem_tensor((self.keys_per_thread,), Uint64)
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(**compile_key_fields)
smem = cutlass.utils.SmemAllocator()
storage = smem.allocate(self.shared_storage, 8)
committed_count_smem = storage.committed_count.data_ptr()
prefix_smem = storage.prefix_s.data_ptr()
for i in range(tid, self.topk, self.tb_size):
output_row[i] = Int32(-1)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
if dcp_world_size <= 1:
return []
topk = vllm_config.model_config.hf_config.index_topk
if topk <= 0:
return []
return self._trace_dispatch(self.dispatch)(
topk=topk,
num_candidates=topk * dcp_world_size,
)
for key_idx in cutlass.range_constexpr(self.keys_per_thread):
col = tid + Int32(key_idx * self.tb_size)
score = Float32(input_row[col, 0])
token_id = Int32(input_row[col, 1])
keys[key_idx] = self._stable_key(score, token_id)
if tid == Int32(0):
committed_count_smem.store(Int32(0))
prefix_smem.store(Uint64(0))
cute.arch.sync_threads()
step = Int32(0)
finished = Int32(0)
while finished == Int32(0) and step < Int32(self.radix_passes - 1):
finished = self._radix_pass(
keys,
output_row,
storage,
tid,
step,
self.radix_bits,
False,
)
step += Int32(1)
if finished == Int32(0):
self._radix_pass(
keys,
output_row,
storage,
tid,
Int32(self.radix_passes - 1),
self.final_radix_bits,
True,
)
@cache
@staticmethod
def compile(topk: int, num_candidates: int):
def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
num_rows = cute.sym_int()
gathered = cute.runtime.make_fake_tensor(
Float32,
(num_rows, num_candidates, 2),
(num_rows, compile_key.num_candidates, 2),
stride=(cute.sym_int64(divisibility=2), 2, 1),
assumed_align=8,
)
out = make_fake_tensor(Int32, (num_rows, topk), divisibility=1)
kernel = StableTopKFromGatheredCandidatesKernel(topk, num_candidates)
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
return cute.compile(
kernel,
gathered,
out,
stream,
options="--enable-tvm-ffi",
out = make_fake_tensor(
Int32,
(num_rows, compile_key.topk),
divisibility=1,
)
return gathered, out
@cutedsl_kernel_launcher
def __call__(
self,
gathered: torch.Tensor,
out: torch.Tensor,
*,
topk: int,
) -> CuTeDSLLaunchSpec[CompileKey]:
compile_key = self.dispatch(topk=topk, num_candidates=gathered.shape[1])
return (
compile_key,
(gathered, out),
{
"gathered_shape": tuple(gathered.shape),
"out_shape": tuple(out.shape),
"topk": topk,
},
)
_PACK_DCP_TOPK_CANDIDATES_KERNEL = PackDCPTopkCandidatesKernel()
_STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL = StableTopKFromGatheredCandidatesKernel()
@@ -768,7 +768,8 @@ class SparseAttnIndexer(CustomOp):
# DCP scalars are constant for the run; resolve them here (config is set
# during model construction) and pass them into the custom op, rather
# than threading them through per-step metadata.
parallel_config = get_current_vllm_config().parallel_config
vllm_config = get_current_vllm_config()
parallel_config = vllm_config.parallel_config
self._parallel_config = parallel_config
self.dcp_world_size = parallel_config.decode_context_parallel_size
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
@@ -780,6 +781,28 @@ class SparseAttnIndexer(CustomOp):
"the current vLLM environment."
)
if vllm_config.kernel_config.enable_jit_warmup:
from vllm.v1.attention.ops.common import (
_PACK_SEQ_TRITON_KERNEL,
_UNPACK_SEQ_TRITON_KERNEL,
)
pack_dtype = torch.uint8 if use_fp4_cache else current_platform.fp8_dtype()
_PACK_SEQ_TRITON_KERNEL.register_warmup(
dtype=pack_dtype,
pad_value=0 if use_fp4_cache else -float("inf"),
)
_UNPACK_SEQ_TRITON_KERNEL.register_warmup()
if self.dcp_world_size > 1 and current_platform.is_cuda() and has_cutedsl():
from vllm.model_executor.kernels.attention.dsa.dcp_indexer_cutedsl import ( # noqa: E501
_PACK_DCP_TOPK_CANDIDATES_KERNEL,
_STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL,
)
_PACK_DCP_TOPK_CANDIDATES_KERNEL.register_warmup()
_STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL.register_warmup()
@property
def cp_kv_cache_interleave_size(self) -> int:
"""With PD+DCP, the real value isn't known until block_size is finalized,
+37 -7
View File
@@ -670,6 +670,34 @@ class VllmJitKernel(Generic[CompileKeyT], ABC):
self.compile(compile_key)
def _same_value(left: Any, right: Any) -> bool:
"""True if two registration values are the same object or compare equal.
Identity is checked first so shared singletons (e.g. ``vllm_config``, torch
dtypes) short-circuit before any potentially deep or non-boolean ``__eq__``.
"""
if left is right:
return True
try:
return bool(left == right)
except (TypeError, ValueError, RuntimeError):
return False
def _same_registration(
left: tuple[tuple[Any, ...], dict[str, Any]],
right: tuple[tuple[Any, ...], dict[str, Any]],
) -> bool:
"""True if two ``(args, kwargs)`` registrations are equivalent."""
left_args, left_kwargs = left
right_args, right_kwargs = right
if len(left_args) != len(right_args) or left_kwargs.keys() != right_kwargs.keys():
return False
return all(
_same_value(x, y) for x, y in zip(left_args, right_args, strict=True)
) and all(_same_value(left_kwargs[k], right_kwargs[k]) for k in left_kwargs)
class JitWarmupRegistry:
"""Collect and compile JIT kernels selected during runner setup."""
@@ -713,13 +741,15 @@ class JitWarmupRegistry:
kwargs: dict[str, Any],
) -> None:
registrations = self._registrations.setdefault(kernel, [])
if (
not args
and not kwargs
and any(
not registered_args and not registered_kwargs
for registered_args, registered_kwargs in registrations
)
# Every layer of a deep model appends an identical (args, kwargs) here
# (e.g. a 61-layer DSA model registers the same pack (dtype, pad_value)
# 61 times); each expands to the same compile keys, so tracing
# get_warmup_keys once per distinct registration is sufficient. Dedup on
# identity-or-equality -- identity short-circuits shared singletons like
# vllm_config before any deep __eq__.
if any(
_same_registration(registered, (args, kwargs))
for registered in registrations
):
return
registrations.append((args, kwargs))
@@ -0,0 +1,80 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import abstractmethod
from collections.abc import Callable, Mapping
from functools import wraps
from typing import Any, ClassVar, Generic, TypeAlias, TypeVar
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel
DEFAULT_CUTEDSL_COMPILE_OPTIONS = "--enable-tvm-ffi"
CompileKeyT = TypeVar("CompileKeyT")
CuTeDSLLaunchSpec: TypeAlias = tuple[
CompileKeyT,
tuple[Any, ...],
Mapping[str, Any] | None,
]
def cutedsl_fake_stream(*, use_tvm_ffi_env_stream: bool = True) -> Any:
import cutlass.cute as cute
return cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=use_tvm_ffi_env_stream)
def compile_cutedsl(
entry: Callable[..., Any],
*args: Any,
options: str = DEFAULT_CUTEDSL_COMPILE_OPTIONS,
use_tvm_ffi_env_stream: bool = True,
) -> Any:
import cutlass.cute as cute
return cute.compile(
entry,
*args,
cutedsl_fake_stream(use_tvm_ffi_env_stream=use_tvm_ffi_env_stream),
options=options,
)
class VllmCuTeDSLJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
"""CuTeDSL owner whose compiled executor is shared by warmup and runtime."""
kernel: ClassVar[Any]
@abstractmethod
def warmup_inputs(self, compile_key: CompileKeyT) -> tuple[Any, ...]:
"""Return fake arguments that compile one executor specialization."""
raise NotImplementedError
def compile(self, compile_key: CompileKeyT) -> None:
if compile_key in self._compiled_cache:
return
self._compiled_cache[compile_key] = compile_cutedsl(
self.kernel(compile_key),
*self.warmup_inputs(compile_key),
)
def cutedsl_kernel_launcher(
call_fn: Callable[..., CuTeDSLLaunchSpec[CompileKeyT]],
) -> Callable[..., Any]:
"""Invoke a cached CuTeDSL executor from a declarative ``__call__``."""
@wraps(call_fn)
def wrapper(
self: VllmCuTeDSLJitKernel[CompileKeyT],
*args: Any,
**kwargs: Any,
) -> Any:
compile_key, launch_args, runtime_context = call_fn(self, *args, **kwargs)
executor = self._get_or_compile(
compile_key,
runtime_context=runtime_context,
)
return executor(*launch_args)
return wrapper
@@ -106,8 +106,16 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
self._warming = False
@cached_property
def _kernel_param_names(self) -> frozenset[str]:
return frozenset(self.kernel.arg_names)
def _kernel_arg_names(self) -> tuple[str, ...]:
arg_names = getattr(self.kernel, "arg_names", None)
if arg_names is not None:
return tuple(arg_names)
wrapped = getattr(self.kernel, "func", None)
if wrapped is not None:
return tuple(inspect.signature(wrapped).parameters)
raise TypeError(
f"Cannot inspect kernel parameters for {type(self.kernel).__name__}"
)
def launch(
self,
@@ -116,14 +124,22 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
/,
**kwargs: Any,
) -> Any:
runtime_launcher = kwargs.pop("_runtime_launcher", None)
runtime_launcher_arg_count = kwargs.pop("_runtime_launcher_arg_count", 0)
for name, value in inputs.items():
target = name if name in self._kernel_param_names else f"{name}_ptr"
if target in self._kernel_param_names and target not in kwargs:
target = name if name in self._kernel_arg_names else f"{name}_ptr"
if target in self._kernel_arg_names and target not in kwargs:
kwargs[target] = value
if self._warming:
warmup = getattr(self.kernel, "warmup", None)
assert warmup is not None
return warmup(grid=(1,), **kwargs)
if runtime_launcher is not None:
regular_args = [
kwargs.pop(name)
for name in self._kernel_arg_names[:runtime_launcher_arg_count]
]
return runtime_launcher(self.kernel, grid, *regular_args, **kwargs)
return self.kernel[grid](**kwargs)
+259 -103
View File
@@ -1,63 +1,157 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any
import torch
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
kernel_launcher,
)
from vllm.triton_utils import tl, triton
@triton.jit
def _pack_seq_kernel(
x_ptr, # [N, D]
out_ptr, # [B, Lmax, D]
lengths_ptr, # *i32, [B]
N: tl.constexpr,
D: tl.constexpr,
Lmax: tl.constexpr,
PAD_VALUE: tl.constexpr,
PAD_IS_UINT8: tl.constexpr,
BLOCK_T: tl.constexpr, # timesteps per program
BLOCK_D: tl.constexpr, # features per program
):
pid_b = tl.program_id(0) # batch id
pid_t = tl.program_id(1) # block over time dimension
pid_d = tl.program_id(2) # block over feature dimension
off_t = pid_t * BLOCK_T + tl.arange(0, BLOCK_T) # [BLOCK_T]
off_d = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) # [BLOCK_D]
class PackSeqTritonKernel(VllmTritonJitKernel["PackSeqTritonKernel.CompileKey"]):
@dataclass(frozen=True)
class CompileKey:
dtype: torch.dtype
pad_value: float | int
pad_is_uint8: bool
block_t: int
block_d: int
# Compute start index and sequence length from cumulative lengths
in_start = 0
for i in range(pid_b):
in_start += tl.load(lengths_ptr + i)
seq_len = tl.load(lengths_ptr + pid_b)
# N is unused; D and Lmax only affect runtime addressing and masks. The
# compile-time tile shape remains controlled by BLOCK_T and BLOCK_D.
@staticmethod
@triton.jit(do_not_specialize=["N", "D", "Lmax"])
def kernel(
x_ptr, # [N, D]
out_ptr, # [B, Lmax, D]
lengths_ptr, # *i32, [B]
N,
D,
Lmax,
PAD_VALUE: tl.constexpr,
PAD_IS_UINT8: tl.constexpr,
BLOCK_T: tl.constexpr, # timesteps per program
BLOCK_D: tl.constexpr, # features per program
):
pid_b = tl.program_id(0) # batch id
pid_t = tl.program_id(1) # block over time dimension
pid_d = tl.program_id(2) # block over feature dimension
off_t = pid_t * BLOCK_T + tl.arange(0, BLOCK_T) # [BLOCK_T]
off_d = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) # [BLOCK_D]
# valid time positions for this block
t_mask = off_t < Lmax
# Compute start index and sequence length from cumulative lengths
in_start = 0
for i in range(pid_b):
in_start += tl.load(lengths_ptr + i)
seq_len = tl.load(lengths_ptr + pid_b)
# compute input row indices for valid (b, t)
in_row = in_start + off_t
valid_row = (off_t < seq_len) & t_mask
# valid time positions for this block
t_mask = off_t < Lmax
# Pointers
# x_ptr: row-major [N, D]
x_row_ptr = x_ptr + in_row[:, None] * D + off_d[None, :]
# compute input row indices for valid (b, t)
in_row = in_start + off_t
valid_row = (off_t < seq_len) & t_mask
# out_ptr: row-major [B, Lmax, D]
out_row_ptr = out_ptr + (pid_b * Lmax + off_t)[:, None] * D + off_d[None, :]
# Pointers
# x_ptr: row-major [N, D]
x_row_ptr = x_ptr + in_row[:, None] * D + off_d[None, :]
# Initialize with PAD. PAD_IS_UINT8 selects the pad tensor's dtype so
# integer-typed outputs (e.g. MXFP4 packed nibbles, ue8m0 scale bytes)
# get an exact-byte pad rather than going through an fp32→uint8 cast
# that's implementation-defined outside of value 0.
d_mask = off_d[None, :] < D
if PAD_IS_UINT8:
pad_vals = tl.full([BLOCK_T, BLOCK_D], PAD_VALUE, tl.uint8)
else:
pad_vals = tl.full([BLOCK_T, BLOCK_D], PAD_VALUE, tl.float32)
tl.store(out_row_ptr, pad_vals, mask=t_mask[:, None] & d_mask)
# out_ptr: row-major [B, Lmax, D]
out_row_ptr = out_ptr + (pid_b * Lmax + off_t)[:, None] * D + off_d[None, :]
# Load & write only where within seq_len
x_vals = tl.load(x_row_ptr, mask=valid_row[:, None] & d_mask)
tl.store(out_row_ptr, x_vals, mask=valid_row[:, None] & d_mask)
# Initialize with PAD. PAD_IS_UINT8 selects the pad tensor's dtype so
# integer-typed outputs (e.g. MXFP4 packed nibbles, ue8m0 scale bytes)
# get an exact-byte pad rather than going through an fp32→uint8 cast
# that's implementation-defined outside of value 0.
d_mask = off_d[None, :] < D
if PAD_IS_UINT8:
pad_vals = tl.full([BLOCK_T, BLOCK_D], PAD_VALUE, tl.uint8)
else:
pad_vals = tl.full([BLOCK_T, BLOCK_D], PAD_VALUE, tl.float32)
tl.store(out_row_ptr, pad_vals, mask=t_mask[:, None] & d_mask)
# Load & write only where within seq_len
x_vals = tl.load(x_row_ptr, mask=valid_row[:, None] & d_mask)
tl.store(out_row_ptr, x_vals, mask=valid_row[:, None] & d_mask)
def dispatch( # type: ignore[override]
self,
*,
dtype: torch.dtype,
pad_value: float | int,
**compile_key_fields: int,
) -> CompileKey:
is_uint8 = dtype == torch.uint8
return self.CompileKey(
**compile_key_fields,
dtype=dtype,
pad_value=int(pad_value) if is_uint8 else float(pad_value),
pad_is_uint8=is_uint8,
)
def get_warmup_keys(
self,
*,
dtype: torch.dtype,
pad_value: float | int,
) -> list[CompileKey]:
return self._trace_dispatch(self.dispatch)(
dtype=dtype,
pad_value=pad_value,
block_t=64,
block_d=64,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
return dict(
x_reshaped=TritonWarmupTensor(compile_key.dtype),
out=TritonWarmupTensor(compile_key.dtype),
lengths=TritonWarmupTensor(torch.int32),
N=1,
D=1,
Lmax=1,
pad_value=compile_key.pad_value,
pad_is_uint8=compile_key.pad_is_uint8,
block_t=compile_key.block_t,
block_d=compile_key.block_d,
)
@kernel_launcher
def __call__(
self,
x_reshaped: torch.Tensor,
out: torch.Tensor,
lengths: torch.Tensor,
*,
N: int,
D: int,
Lmax: int,
pad_value: float | int,
pad_is_uint8: bool,
block_t: int,
block_d: int,
) -> LaunchSpec:
grid = (
lengths.shape[0],
triton.cdiv(Lmax, block_t),
triton.cdiv(D, block_d),
)
return grid, dict(
x_ptr=x_reshaped,
lengths_ptr=lengths,
PAD_VALUE=pad_value,
PAD_IS_UINT8=pad_is_uint8,
BLOCK_T=block_t,
BLOCK_D=block_d,
num_warps=4,
num_stages=2,
)
def pack_seq_triton(
@@ -109,20 +203,18 @@ def pack_seq_triton(
out = torch.empty((B, Lmax, D), device=x.device, dtype=x.dtype)
grid = (B, triton.cdiv(Lmax, block_t), triton.cdiv(D, block_d))
_pack_seq_kernel[grid](
lengths = lengths.int()
_PACK_SEQ_TRITON_KERNEL(
x_reshaped,
out,
lengths.int(),
N,
D,
Lmax,
PAD_VALUE=pad_constexpr,
PAD_IS_UINT8=is_uint8,
BLOCK_T=block_t,
BLOCK_D=block_d,
num_warps=4,
num_stages=2,
lengths,
N=N,
D=D,
Lmax=Lmax,
pad_value=pad_constexpr,
pad_is_uint8=is_uint8,
block_t=block_t,
block_d=block_d,
)
if len(original_shape) > 2:
@@ -131,47 +223,109 @@ def pack_seq_triton(
return out
@triton.jit
def _unpack_seq_triton_kernel(
packed_ptr, # [B, Lmax, D]
out_ptr, # [N, D]
lengths_ptr, # *i32, [B]
B: tl.constexpr,
Lmax: tl.constexpr,
D: tl.constexpr,
BLOCK_T: tl.constexpr, # timesteps per program
BLOCK_D: tl.constexpr, # features per program
):
pid_b = tl.program_id(0) # batch id
pid_t = tl.program_id(1) # block over time dimension
pid_d = tl.program_id(2) # block over feature dimension
off_t = pid_t * BLOCK_T + tl.arange(0, BLOCK_T) # [BLOCK_T]
off_d = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) # [BLOCK_D]
class UnpackSeqTritonKernel(VllmTritonJitKernel["UnpackSeqTritonKernel.CompileKey"]):
@dataclass(frozen=True)
class CompileKey:
dtype: torch.dtype
block_t: int
block_d: int
# bounds: compute start from cumulative lengths
in_start = 0
for i in range(pid_b):
in_start += tl.load(lengths_ptr + i)
seq_len = tl.load(lengths_ptr + pid_b)
# B, Lmax, and D only affect runtime addressing and masks. The compile-time
# tile shape remains controlled by BLOCK_T and BLOCK_D.
@staticmethod
@triton.jit(do_not_specialize=["B", "Lmax", "D"])
def kernel(
packed_ptr, # [B, Lmax, D]
out_ptr, # [N, D]
lengths_ptr, # *i32, [B]
B,
Lmax,
D,
BLOCK_T: tl.constexpr, # timesteps per program
BLOCK_D: tl.constexpr, # features per program
):
pid_b = tl.program_id(0) # batch id
pid_t = tl.program_id(1) # block over time dimension
pid_d = tl.program_id(2) # block over feature dimension
off_t = pid_t * BLOCK_T + tl.arange(0, BLOCK_T) # [BLOCK_T]
off_d = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) # [BLOCK_D]
# valid time positions for this block
t_mask = off_t < Lmax
valid_row = (off_t < seq_len) & t_mask
# bounds: compute start from cumulative lengths
in_start = 0
for i in range(pid_b):
in_start += tl.load(lengths_ptr + i)
seq_len = tl.load(lengths_ptr + pid_b)
# compute output row indices for valid (b, t)
out_row = in_start + off_t
# valid time positions for this block
t_mask = off_t < Lmax
valid_row = (off_t < seq_len) & t_mask
# Pointers
# packed_ptr: row-major [B, Lmax, D]
packed_row_ptr = packed_ptr + (pid_b * Lmax + off_t)[:, None] * D + off_d[None, :]
# compute output row indices for valid (b, t)
out_row = in_start + off_t
# out_ptr: row-major [N, D]
out_row_ptr = out_ptr + out_row[:, None] * D + off_d[None, :]
# Pointers
# packed_ptr: row-major [B, Lmax, D]
packed_row_ptr = (
packed_ptr + (pid_b * Lmax + off_t)[:, None] * D + off_d[None, :]
)
# Load from packed tensor and store to output
d_mask = off_d[None, :] < D
packed_vals = tl.load(packed_row_ptr, mask=valid_row[:, None] & d_mask)
tl.store(out_row_ptr, packed_vals, mask=valid_row[:, None] & d_mask)
# out_ptr: row-major [N, D]
out_row_ptr = out_ptr + out_row[:, None] * D + off_d[None, :]
# Load from packed tensor and store to output
d_mask = off_d[None, :] < D
packed_vals = tl.load(packed_row_ptr, mask=valid_row[:, None] & d_mask)
tl.store(out_row_ptr, packed_vals, mask=valid_row[:, None] & d_mask)
def dispatch( # type: ignore[override]
self,
*,
dtype: torch.dtype,
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(**compile_key_fields, dtype=dtype)
def get_warmup_keys(self) -> list[CompileKey]:
return self._trace_dispatch(self.dispatch)(
dtype=torch.int32,
block_t=64,
block_d=64,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
return dict(
packed_reshaped=TritonWarmupTensor(compile_key.dtype),
out=TritonWarmupTensor(compile_key.dtype),
lengths=TritonWarmupTensor(torch.int32),
B=1,
Lmax=1,
D=1,
block_t=compile_key.block_t,
block_d=compile_key.block_d,
)
@kernel_launcher
def __call__(
self,
packed_reshaped: torch.Tensor,
out: torch.Tensor,
lengths: torch.Tensor,
*,
B: int,
Lmax: int,
D: int,
block_t: int,
block_d: int,
) -> LaunchSpec:
grid = (B, triton.cdiv(Lmax, block_t), triton.cdiv(D, block_d))
return grid, dict(
packed_ptr=packed_reshaped,
lengths_ptr=lengths,
BLOCK_T=block_t,
BLOCK_D=block_d,
num_warps=4,
num_stages=2,
)
def unpack_seq_triton(
@@ -209,18 +363,16 @@ def unpack_seq_triton(
out = torch.empty((N, D), device=packed_tensor.device, dtype=packed_tensor.dtype)
grid = (B, triton.cdiv(Lmax, block_t), triton.cdiv(D, block_d))
_unpack_seq_triton_kernel[grid](
lengths = lengths.int()
_UNPACK_SEQ_TRITON_KERNEL(
packed_reshaped,
out,
lengths.int(),
B,
Lmax,
D,
BLOCK_T=block_t,
BLOCK_D=block_d,
num_warps=4,
num_stages=2,
lengths,
B=B,
Lmax=Lmax,
D=D,
block_t=block_t,
block_d=block_d,
)
# Reshape output back to original dimensions (except first dimension)
@@ -229,3 +381,7 @@ def unpack_seq_triton(
out = out.reshape(output_shape)
return out
_PACK_SEQ_TRITON_KERNEL = PackSeqTritonKernel()
_UNPACK_SEQ_TRITON_KERNEL = UnpackSeqTritonKernel()
+269 -97
View File
@@ -6,7 +6,8 @@ from __future__ import annotations
import functools
from collections.abc import Callable
from typing import TYPE_CHECKING, Protocol
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Protocol
import torch
import torch.distributed as dist
@@ -15,6 +16,16 @@ import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.distributed import get_dcp_group
from vllm.logger import init_logger
from vllm.model_executor.warmup.jit_warmup import (
WarmupIntRange,
)
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
kernel_launcher,
triton_scalar_specialization_rep,
)
from vllm.triton_utils import tl, triton
from vllm.v1.attention.ops.cp_common import (
DirectCPWorkspace,
@@ -65,93 +76,254 @@ def mask_dcp_empty_shards_(
# AG + RS/AR implementation
@triton.jit
def _correct_attn_cp_out_kernel(
outputs_ptr,
new_output_ptr,
lses_ptr,
vlse_ptr,
outputs_stride_B,
outputs_stride_H,
outputs_stride_D,
lses_stride_N,
lses_stride_B,
lses_stride_H,
lse_idx,
HEAD_DIM: tl.constexpr,
N_ROUNDED: tl.constexpr,
IS_BASE_E: tl.constexpr,
):
"""
Apply the all-gathered lses to correct each local rank's attention
output. we still need perform a cross-rank reduction to obtain the
final attention output.
class CorrectAttnCPOutKernel(VllmTritonJitKernel["CorrectAttnCPOutKernel.CompileKey"]):
@dataclass(frozen=True)
class CompileKey:
output_dtype: torch.dtype
lse_dtype: torch.dtype
outputs_stride_b: int
outputs_stride_h: int
outputs_stride_d: int
lses_stride_n: int
lses_stride_b: int
lses_stride_h: int
lse_idx: int
head_dim: int
n_rounded: int
is_base_e: bool
Args:
outputs_ptr (triton.PointerType):
Pointer to input tensor of shape [ B, H, D ]
lses_ptr (triton.PointerType):
Pointer to input tensor of shape [ N, B, H ]
new_output_ptr (triton.PointerType):
Pointer to output tensor of shape [ B, H, D ]
vlse_ptr (triton.PointerType):
Pointer to output tensor of shape [ B, H ]
"""
batch_idx = tl.program_id(axis=0).to(tl.int64)
head_idx = tl.program_id(axis=1).to(tl.int64)
d_offsets = tl.arange(0, HEAD_DIM)
num_n_offsets = tl.arange(0, N_ROUNDED)
@staticmethod
@triton.jit
def kernel(
outputs_ptr,
new_output_ptr,
lses_ptr,
vlse_ptr,
outputs_stride_B,
outputs_stride_H,
outputs_stride_D,
lses_stride_N,
lses_stride_B,
lses_stride_H,
lse_idx,
HEAD_DIM: tl.constexpr,
N_ROUNDED: tl.constexpr,
IS_BASE_E: tl.constexpr,
):
"""
Apply the all-gathered lses to correct each local rank's attention
output. we still need perform a cross-rank reduction to obtain the
final attention output.
# shape = [N]
lse_offsets = (
num_n_offsets * lses_stride_N
+ batch_idx * lses_stride_B
+ head_idx * lses_stride_H
)
Args:
outputs_ptr (triton.PointerType):
Pointer to input tensor of shape [ B, H, D ]
lses_ptr (triton.PointerType):
Pointer to input tensor of shape [ N, B, H ]
new_output_ptr (triton.PointerType):
Pointer to output tensor of shape [ B, H, D ]
vlse_ptr (triton.PointerType):
Pointer to output tensor of shape [ B, H ]
"""
batch_idx = tl.program_id(axis=0).to(tl.int64)
head_idx = tl.program_id(axis=1).to(tl.int64)
d_offsets = tl.arange(0, HEAD_DIM)
num_n_offsets = tl.arange(0, N_ROUNDED)
# calc final lse
lse = tl.load(lses_ptr + lse_offsets).to(tl.float32)
lse = tl.where((lse != lse) | (lse == float("inf")), -float("inf"), lse)
lse_max = tl.max(lse, axis=0)
lse_max = tl.where(lse_max == -float("inf"), 0, lse_max)
lse -= lse_max
if IS_BASE_E:
lse_exp = tl.exp(lse)
lse_acc = tl.sum(lse_exp, axis=0)
lse = tl.log(lse_acc)
else:
lse_exp = tl.exp2(lse)
lse_acc = tl.sum(lse_exp, axis=0)
lse = tl.log2(lse_acc)
lse += lse_max
# shape = [N]
lse_offsets = (
num_n_offsets * lses_stride_N
+ batch_idx * lses_stride_B
+ head_idx * lses_stride_H
)
lse_offsets = batch_idx * lses_stride_B + head_idx * lses_stride_H
tl.store(vlse_ptr + lse_offsets, lse)
# calc final lse
lse = tl.load(lses_ptr + lse_offsets).to(tl.float32)
lse = tl.where((lse != lse) | (lse == float("inf")), -float("inf"), lse)
lse_max = tl.max(lse, axis=0)
lse_max = tl.where(lse_max == -float("inf"), 0, lse_max)
lse -= lse_max
if IS_BASE_E:
lse_exp = tl.exp(lse)
lse_acc = tl.sum(lse_exp, axis=0)
lse = tl.log(lse_acc)
else:
lse_exp = tl.exp2(lse)
lse_acc = tl.sum(lse_exp, axis=0)
lse = tl.log2(lse_acc)
lse += lse_max
# shape = [D]
output_offsets = (
batch_idx * outputs_stride_B
+ head_idx * outputs_stride_H
+ d_offsets * outputs_stride_D
)
lse_offsets = batch_idx * lses_stride_B + head_idx * lses_stride_H
tl.store(vlse_ptr + lse_offsets, lse)
# correct output
lse_offset = (
lse_idx * lses_stride_N + batch_idx * lses_stride_B + head_idx * lses_stride_H
)
lse_tmp = tl.load(lses_ptr + lse_offset).to(tl.float32)
lse_finally = lse_tmp - lse
lse_finally = tl.where(
(lse_finally != lse_finally) | (lse_finally == float("inf")),
-float("inf"),
lse_finally,
)
factor = tl.exp(lse_finally) if IS_BASE_E else tl.exp2(lse_finally)
output = tl.load(outputs_ptr + output_offsets)
output = output * factor
output = tl.where(factor == 0.0, 0.0, output)
# shape = [D]
output_offsets = (
batch_idx * outputs_stride_B
+ head_idx * outputs_stride_H
+ d_offsets * outputs_stride_D
)
tl.store(new_output_ptr + output_offsets, output)
# correct output
lse_offset = (
lse_idx * lses_stride_N
+ batch_idx * lses_stride_B
+ head_idx * lses_stride_H
)
lse_tmp = tl.load(lses_ptr + lse_offset).to(tl.float32)
lse_finally = lse_tmp - lse
lse_finally = tl.where(
(lse_finally != lse_finally) | (lse_finally == float("inf")),
-float("inf"),
lse_finally,
)
factor = tl.exp(lse_finally) if IS_BASE_E else tl.exp2(lse_finally)
output = tl.load(outputs_ptr + output_offsets)
output = output * factor
output = tl.where(factor == 0.0, 0.0, output)
tl.store(new_output_ptr + output_offsets, output)
def dispatch( # type: ignore[override]
self,
*,
output_dtype: torch.dtype,
lse_dtype: torch.dtype,
num_tokens: int,
num_heads: int,
head_dim: int,
n_rounded: int,
lse_idx: int,
is_base_e: bool,
) -> CompileKey:
# Warm the contiguous [B, H, D] output and [N, B, H] LSE layouts used
# by the production collective path. Runtime still forwards real
# strides so unsupported layouts remain correct, but may compile on
# first use under the JIT monitor.
outputs_stride_d = 1
outputs_stride_h = head_dim
outputs_stride_b = num_heads * head_dim
lses_stride_h = 1
lses_stride_b = num_heads
lses_stride_n = num_tokens * num_heads
return self.CompileKey(
output_dtype=output_dtype,
lse_dtype=lse_dtype,
outputs_stride_b=triton_scalar_specialization_rep(outputs_stride_b),
outputs_stride_h=triton_scalar_specialization_rep(outputs_stride_h),
outputs_stride_d=triton_scalar_specialization_rep(outputs_stride_d),
lses_stride_n=triton_scalar_specialization_rep(lses_stride_n),
lses_stride_b=triton_scalar_specialization_rep(lses_stride_b),
lses_stride_h=triton_scalar_specialization_rep(lses_stride_h),
lse_idx=triton_scalar_specialization_rep(lse_idx),
head_dim=head_dim,
n_rounded=n_rounded,
is_base_e=is_base_e,
)
def get_warmup_keys(
self,
vllm_config: Any,
*,
output_dtype: torch.dtype | None = None,
num_heads: int | None = None,
head_dim: int | None = None,
is_base_e: bool | tuple[bool, ...] = (False, True),
) -> list[CompileKey]:
from vllm.model_executor.layers.attention.mla_attention import (
get_mla_dims,
)
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
max_tokens = min(
16,
vllm_config.scheduler_config.max_num_batched_tokens,
)
hf_config = vllm_config.model_config.hf_config
if num_heads is None:
num_heads = (
hf_config.num_attention_heads
// vllm_config.parallel_config.tensor_parallel_size
)
if head_dim is None:
head_dim = get_mla_dims(vllm_config.model_config).v_head_dim
if output_dtype is None:
output_dtype = vllm_config.model_config.dtype
if dcp_world_size <= 1 or max_tokens <= 0 or num_heads <= 0:
return []
return self._trace_dispatch(self.dispatch)(
output_dtype=output_dtype,
lse_dtype=torch.float32,
num_tokens=WarmupIntRange(1, max_tokens + 1),
num_heads=num_heads,
head_dim=head_dim,
n_rounded=dcp_world_size,
lse_idx=WarmupIntRange(0, dcp_world_size),
is_base_e=is_base_e,
)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
output_ptr = TritonWarmupTensor(
compile_key.output_dtype,
shape=(1, 1, compile_key.head_dim),
strides=(
compile_key.outputs_stride_b,
compile_key.outputs_stride_h,
compile_key.outputs_stride_d,
),
)
lse_ptr = TritonWarmupTensor(
compile_key.lse_dtype,
shape=(compile_key.n_rounded, 1, 1),
strides=(
compile_key.lses_stride_n,
compile_key.lses_stride_b,
compile_key.lses_stride_h,
),
)
return dict(
outputs=output_ptr,
new_output=output_ptr,
lses=lse_ptr,
vlse=lse_ptr,
lse_idx=compile_key.lse_idx,
ctx=None,
is_base_e=compile_key.is_base_e,
)
@kernel_launcher
def __call__(
self,
outputs: torch.Tensor,
new_output: torch.Tensor,
lses: torch.Tensor,
vlse: torch.Tensor,
lse_idx: int,
ctx: Any,
*,
is_base_e: bool,
) -> LaunchSpec:
num_tokens, num_heads, head_dim = outputs.shape
n_rounded = lses.shape[0]
outputs_stride_b, outputs_stride_h, outputs_stride_d = outputs.stride()
lses_stride_n, lses_stride_b, lses_stride_h = lses.stride()
grid = (num_tokens, num_heads, 1)
return grid, dict(
outputs_stride_B=outputs_stride_b,
outputs_stride_H=outputs_stride_h,
outputs_stride_D=outputs_stride_d,
lses_stride_N=lses_stride_n,
lses_stride_B=lses_stride_b,
lses_stride_H=lses_stride_h,
HEAD_DIM=head_dim,
N_ROUNDED=n_rounded,
IS_BASE_E=is_base_e,
_runtime_launcher=None if self._warming else ctx.call_kernel,
# CPTritonContext caches the non-constexpr positional prefix; derive
# its length so adding/reordering a kernel arg cannot silently
# misalign the cached replay path.
_runtime_launcher_arg_count=len(self.kernel.arg_names)
- len(self.kernel.constexprs),
)
class CPTritonContext:
@@ -202,14 +374,12 @@ def correct_attn_out(
f"got {tuple(lses.shape)}"
)
B, H, D = out.shape
N = lses.shape[0]
B, H, _ = out.shape
# Strides after we normalized shapes to 3-D views. The kernel computes
# offsets for `vlse_ptr` using lses_stride_B/H, so the output buffer must
# have the same B/H stride layout as a slice of `lses`.
o_sB, o_sH, o_sD = out.stride()
l_sN, l_sB, l_sH = lses.stride()
_, l_sB, l_sH = lses.stride()
# Allocate LSE with the same B/H strides as `lses` so writes land correctly
# even when `lses` is a non-contiguous view (e.g., 4-D to 3-D squeeze).
@@ -217,24 +387,15 @@ def correct_attn_out(
(B, H), (l_sB, l_sH), device=lses.device, dtype=lses.dtype
)
# Kernel launch config
grid = (B, H, 1)
regular_args = (
_CORRECT_ATTN_CP_OUT_KERNEL(
out,
out,
lses,
lse,
o_sB,
o_sH,
o_sD,
l_sN,
l_sB,
l_sH,
cp_rank,
ctx,
is_base_e=is_lse_base_on_e,
)
const_args = {"HEAD_DIM": D, "N_ROUNDED": N, "IS_BASE_E": is_lse_base_on_e}
ctx.call_kernel(_correct_attn_cp_out_kernel, grid, *regular_args, **const_args)
return out, lse
@@ -1250,6 +1411,14 @@ class MLADCPManager:
is_lse_base_on_e,
use_pcp,
)
if vllm_config.kernel_config.enable_jit_warmup and not self.use_a2a:
_CORRECT_ATTN_CP_OUT_KERNEL.register_warmup(
vllm_config,
output_dtype=output_dtype,
num_heads=num_heads,
head_dim=output_head_dim,
is_base_e=is_lse_base_on_e,
)
self.query_gather = (
None
if use_pcp
@@ -1411,3 +1580,6 @@ class MLADCPManager:
local_kv: torch.Tensor,
) -> object:
return self._kv_gather(gathered_kv, local_kv)
_CORRECT_ATTN_CP_OUT_KERNEL = CorrectAttnCPOutKernel()
+46 -28
View File
@@ -11,11 +11,11 @@ import torch
from vllm.distributed import get_dcp_group, get_pcp_group
from vllm.logger import init_logger
from vllm.model_executor.warmup.jit_warmup import (
VllmJitKernel,
)
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonWarmupTensor,
VllmTritonJitKernel,
kernel_launcher,
triton_scalar_specialization_rep,
)
from vllm.triton_utils import tl, triton
@@ -394,7 +394,9 @@ class MultiGroupBlockTable:
return self.block_tables[idx]
class ComputeSlotMappingKernel(VllmJitKernel["ComputeSlotMappingKernel.CompileKey"]):
class ComputeSlotMappingKernel(
VllmTritonJitKernel["ComputeSlotMappingKernel.CompileKey"]
):
triton_block_size = 1024
@dataclass(frozen=True)
@@ -490,37 +492,53 @@ class ComputeSlotMappingKernel(VllmJitKernel["ComputeSlotMappingKernel.CompileKe
def get_warmup_keys(self, **dispatch_kwargs: int) -> list[CompileKey]:
return self._trace_dispatch(self.dispatch)(**dispatch_kwargs)
def compile(self, compile_key: CompileKey) -> None:
warmup = getattr(self.kernel, "warmup", None)
assert warmup is not None
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
int32_ptr = TritonWarmupTensor(torch.int32)
int64_ptr = TritonWarmupTensor(torch.int64)
warmup(
2, # arbitrary, num_tokens in do_not_specialize
2, # arbitrary, max_num_tokens in do_not_specialize
int32_ptr,
int64_ptr,
int32_ptr,
compile_key.block_table_stride,
compile_key.block_size,
int64_ptr,
KV_CACHE_BLOCK_SIZE=compile_key.kv_cache_block_size,
BLOCKS_PER_KV_BLOCK=compile_key.blocks_per_kv_block,
TOTAL_CP_WORLD_SIZE=compile_key.total_cp_world_size,
TOTAL_CP_RANK=compile_key.total_cp_rank,
CP_KV_CACHE_INTERLEAVE_SIZE=compile_key.cp_kv_cache_interleave_size,
PAD_ID=PAD_SLOT_ID,
BLOCK_SIZE=self.triton_block_size,
grid=(2,),
return dict(
num_reqs=1,
num_tokens=2, # arbitrary, in do_not_specialize
max_num_tokens=2, # arbitrary, in do_not_specialize
query_start_loc=int32_ptr,
positions=int64_ptr,
block_table=TritonWarmupTensor(
torch.int32,
shape=(1, compile_key.block_table_stride),
),
block_table_stride=compile_key.block_table_stride,
block_size=compile_key.block_size,
slot_mapping=int64_ptr,
kv_cache_block_size=compile_key.kv_cache_block_size,
blocks_per_kv_block=compile_key.blocks_per_kv_block,
total_cp_world_size=compile_key.total_cp_world_size,
total_cp_rank=compile_key.total_cp_rank,
cp_kv_cache_interleave_size=compile_key.cp_kv_cache_interleave_size,
)
@kernel_launcher
def __call__(
self,
num_reqs: int,
*args: Any,
) -> None:
self.kernel[(num_reqs + 1,)](
*args,
num_tokens: int,
max_num_tokens: int,
query_start_loc: torch.Tensor,
positions: torch.Tensor,
block_table: torch.Tensor,
block_table_stride: int,
block_size: int,
slot_mapping: torch.Tensor,
kv_cache_block_size: int,
blocks_per_kv_block: int,
total_cp_world_size: int,
total_cp_rank: int,
cp_kv_cache_interleave_size: int,
) -> LaunchSpec:
return (num_reqs + 1,), dict(
KV_CACHE_BLOCK_SIZE=kv_cache_block_size,
BLOCKS_PER_KV_BLOCK=blocks_per_kv_block,
TOTAL_CP_WORLD_SIZE=total_cp_world_size,
TOTAL_CP_RANK=total_cp_rank,
CP_KV_CACHE_INTERLEAVE_SIZE=cp_kv_cache_interleave_size,
PAD_ID=PAD_SLOT_ID,
BLOCK_SIZE=self.triton_block_size,
)