diff --git a/.buildkite/test_areas/model_executor.yaml b/.buildkite/test_areas/model_executor.yaml index 1e92462fe6..17818cc152 100644 --- a/.buildkite/test_areas/model_executor.yaml +++ b/.buildkite/test_areas/model_executor.yaml @@ -13,6 +13,8 @@ steps: - vllm/model_executor/warmup - tests/model_executor - tests/model_executor/test_jit_warmup.py + - tests/model_executor/test_jit_warmup_cutedsl_launcher.py + - tests/model_executor/test_jit_warmup_triton_launcher.py - tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py commands: - apt-get update && apt-get install -y curl libsodium23 @@ -41,6 +43,8 @@ steps: - vllm/model_executor/warmup - tests/model_executor - tests/model_executor/test_jit_warmup.py + - tests/model_executor/test_jit_warmup_cutedsl_launcher.py + - tests/model_executor/test_jit_warmup_triton_launcher.py - tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py diff --git a/tests/model_executor/test_jit_warmup.py b/tests/model_executor/test_jit_warmup.py index 29495a510a..b3bbe3231f 100644 --- a/tests/model_executor/test_jit_warmup.py +++ b/tests/model_executor/test_jit_warmup.py @@ -532,10 +532,12 @@ def test_runtime_cache_miss_compiles_and_caches_executor() -> None: def test_registry_records_only_inside_model_setup_context() -> None: registry = JitWarmupRegistry(_config()) kernel = RecordingToyKernel() + config = _config() kernel.register_warmup(3, _config()) with registry.activate(): - kernel.register_warmup(3, _config()) + kernel.register_warmup(3, config) + kernel.register_warmup(3, config) assert len(registry) == 1 assert kernel.compiled == [] diff --git a/tests/model_executor/test_jit_warmup_cutedsl_launcher.py b/tests/model_executor/test_jit_warmup_cutedsl_launcher.py new file mode 100644 index 0000000000..2524c60d2e --- /dev/null +++ b/tests/model_executor/test_jit_warmup_cutedsl_launcher.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from dataclasses import dataclass +from typing import Any + +from vllm.model_executor.warmup import jit_warmup_cutedsl_helper +from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import ( + CuTeDSLLaunchSpec, + VllmCuTeDSLJitKernel, + cutedsl_kernel_launcher, +) + + +class _TestCuTeDSLKernel(VllmCuTeDSLJitKernel["_TestCuTeDSLKernel.CompileKey"]): + @dataclass(frozen=True) + class CompileKey: + variant: int + + @staticmethod + def kernel(compile_key: CompileKey) -> tuple[str, int]: + return "entry", compile_key.variant + + def dispatch(self, *, variant: int) -> CompileKey: + return self.CompileKey(variant=variant) + + def get_warmup_keys(self) -> list[CompileKey]: + return self._trace_dispatch(self.dispatch)(variant=(1, 2)) + + def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]: + return (f"fake-{compile_key.variant}",) + + @cutedsl_kernel_launcher + def __call__( + self, + payload: str, + *, + variant: int, + ) -> CuTeDSLLaunchSpec[CompileKey]: + compile_key = self.dispatch(variant=variant) + return compile_key, (payload,), {"variant": variant} + + +def test_cutedsl_launcher_reuses_compiled_executor(monkeypatch) -> None: + compile_calls: list[tuple[Any, tuple[Any, ...]]] = [] + launch_calls: list[tuple[int, tuple[Any, ...]]] = [] + + def compile_cutedsl(entry: tuple[str, int], *args: Any) -> Any: + compile_calls.append((entry, args)) + variant = entry[1] + + def executor(*runtime_args: Any) -> tuple[int, tuple[Any, ...]]: + launch_calls.append((variant, runtime_args)) + return variant, runtime_args + + return executor + + monkeypatch.setattr( + jit_warmup_cutedsl_helper, + "compile_cutedsl", + compile_cutedsl, + ) + kernel = _TestCuTeDSLKernel() + compile_key = kernel.CompileKey(variant=1) + + kernel.compile(compile_key) + kernel.compile(compile_key) + assert compile_calls == [(("entry", 1), ("fake-1",))] + + assert kernel("runtime", variant=1) == (1, ("runtime",)) + assert kernel("other", variant=2) == (2, ("other",)) + assert compile_calls == [ + (("entry", 1), ("fake-1",)), + (("entry", 2), ("fake-2",)), + ] + assert launch_calls == [(1, ("runtime",)), (2, ("other",))] diff --git a/tests/model_executor/test_jit_warmup_triton_launcher.py b/tests/model_executor/test_jit_warmup_triton_launcher.py new file mode 100644 index 0000000000..2e624cc7f6 --- /dev/null +++ b/tests/model_executor/test_jit_warmup_triton_launcher.py @@ -0,0 +1,128 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from dataclasses import dataclass +from typing import Any + +from vllm.model_executor.warmup.jit_warmup_triton_helper import ( + LaunchSpec, + VllmTritonJitKernel, + kernel_launcher, +) + + +class _FakeTritonKernel: + arg_names = ("first", "second", "CONST") + + def __init__(self) -> None: + self.warmup_calls: list[dict[str, Any]] = [] + + def warmup(self, **kwargs: Any) -> None: + self.warmup_calls.append(kwargs) + + +class _TestTritonKernel(VllmTritonJitKernel["_TestTritonKernel.CompileKey"]): + kernel = _FakeTritonKernel() + + @dataclass(frozen=True) + class CompileKey: + value: int + + def dispatch(self, *, value: int) -> CompileKey: + return self.CompileKey(value=value) + + def get_warmup_keys(self) -> list[CompileKey]: + return self._trace_dispatch(self.dispatch)(value=1) + + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + return dict(first="warmup", second=compile_key.value, runtime_launcher=None) + + @kernel_launcher + def __call__( + self, + first: str, + second: int, + runtime_launcher: Any, + ) -> LaunchSpec: + return (2,), dict( + CONST=7, + _runtime_launcher=runtime_launcher, + _runtime_launcher_arg_count=2, + ) + + +def test_triton_launcher_supports_compile_and_runtime_adapters() -> None: + owner = _TestTritonKernel() + owner.kernel.warmup_calls.clear() + + owner.compile(owner.CompileKey(value=1)) + assert owner.kernel.warmup_calls == [ + {"grid": (1,), "first": "warmup", "second": 1, "CONST": 7} + ] + + runtime_calls: list[tuple[Any, ...]] = [] + + def runtime_launcher(kernel: Any, grid: Any, *args: Any, **kwargs: Any) -> None: + runtime_calls.append((kernel, grid, args, kwargs)) + + owner("runtime", 2, runtime_launcher) + assert runtime_calls == [(owner.kernel, (2,), ("runtime", 2), {"CONST": 7})] + + +def test_triton_launcher_supports_cpu_function_wrappers() -> None: + calls: list[tuple[Any, ...]] = [] + + def kernel(first: str, second: int, CONST: int) -> None: + calls.append((first, second, CONST)) + + class FuncWrapper: + def __init__(self) -> None: + self.func = kernel + + def __getitem__(self, _grid: Any) -> Any: + return self.func + + class TestCpuKernel(_TestTritonKernel): + kernel: Any = FuncWrapper() + + owner = TestCpuKernel() + + owner("runtime", 2, None) + assert calls == [("runtime", 2, 7)] + + +def test_compute_slot_mapping_uses_named_launcher_inputs(monkeypatch) -> None: + from vllm.v1.attention.backends.utils import PAD_SLOT_ID + from vllm.v1.worker.block_table import ComputeSlotMappingKernel + + owner = ComputeSlotMappingKernel() + compile_key = owner.CompileKey( + kv_cache_block_size=16, + blocks_per_kv_block=1, + total_cp_world_size=2, + total_cp_rank=1, + cp_kv_cache_interleave_size=1, + block_table_stride=128, + block_size=16, + ) + launches: list[tuple[Any, ...]] = [] + + def launch(grid: Any, inputs: Any, **kwargs: Any) -> None: + launches.append((grid, inputs, kwargs)) + + monkeypatch.setattr(owner, "launch", launch) + owner.compile(compile_key) + + grid, inputs, kwargs = launches[0] + assert grid == (2,) + assert inputs["num_tokens"] == 2 + assert inputs["block_table_stride"] == 128 + assert kwargs == { + "KV_CACHE_BLOCK_SIZE": 16, + "BLOCKS_PER_KV_BLOCK": 1, + "TOTAL_CP_WORLD_SIZE": 2, + "TOTAL_CP_RANK": 1, + "CP_KV_CACHE_INTERLEAVE_SIZE": 1, + "PAD_ID": PAD_SLOT_ID, + "BLOCK_SIZE": owner.triton_block_size, + } diff --git a/vllm/model_executor/kernels/attention/dsa/dcp_indexer_cutedsl.py b/vllm/model_executor/kernels/attention/dsa/dcp_indexer_cutedsl.py index deb5a87bc0..276e1c465f 100644 --- a/vllm/model_executor/kernels/attention/dsa/dcp_indexer_cutedsl.py +++ b/vllm/model_executor/kernels/attention/dsa/dcp_indexer_cutedsl.py @@ -1,7 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from functools import cache +from dataclasses import dataclass +from typing import Any import cutlass import cutlass.cute as cute @@ -11,6 +12,17 @@ from cutlass import Float32, Int32, Uint32, Uint64 from quack.compile_utils import make_fake_tensor from vllm.cute_utils import recast_val +from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import ( + CuTeDSLLaunchSpec, + VllmCuTeDSLJitKernel, + cutedsl_kernel_launcher, +) +from vllm.model_executor.warmup.jit_warmup_triton_helper import ( + LaunchSpec, + TritonWarmupTensor, + VllmTritonJitKernel, + kernel_launcher, +) from vllm.triton_utils import tl, triton @@ -25,9 +37,7 @@ def stable_topk_from_gathered_candidates_cutedsl( dtype=torch.int32, device=gathered.device, ) - StableTopKFromGatheredCandidatesKernel.compile(topk, gathered.shape[1])( - gathered, out - ) + _STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL(gathered, out, topk=topk) return out @@ -41,380 +51,543 @@ def pack_dcp_topk_candidates_cutedsl( row_starts: torch.Tensor | None, ) -> None: topk = topk_indices.shape[1] - grid = (topk_indices.shape[0], triton.cdiv(topk, 512)) row_starts_arg = row_starts if row_starts is not None else topk_indices - _pack_dcp_topk_candidates_triton_kernel[grid]( + _PACK_DCP_TOPK_CANDIDATES_KERNEL( logits, topk_indices, packed, row_starts_arg, - logits.stride(0), - logits.stride(1), - topk_indices.stride(0), - topk_indices.stride(1), - packed.stride(0), - packed.stride(1), - packed.stride(2), - logits.shape[1], - DCP_RANK=dcp_rank, - DCP_WORLD_SIZE=dcp_world_size, - CP_INTERLEAVE=cp_interleave, - HAS_ROW_STARTS=row_starts is not None, - TOPK=topk, - BLOCK_SIZE=512, - num_warps=8, + logits_stride0=logits.stride(0), + logits_stride1=logits.stride(1), + topk_stride0=topk_indices.stride(0), + topk_stride1=topk_indices.stride(1), + packed_stride0=packed.stride(0), + packed_stride1=packed.stride(1), + packed_stride2=packed.stride(2), + num_cols=logits.shape[1], + dcp_rank=dcp_rank, + dcp_world_size=dcp_world_size, + cp_interleave=cp_interleave, + has_row_starts=row_starts is not None, + topk=topk, + block_size=512, ) -@triton.jit -def _pack_dcp_topk_candidates_triton_kernel( - logits, - topk_indices, - packed, - row_starts, - logits_stride0: tl.constexpr, - logits_stride1: tl.constexpr, - topk_stride0: tl.constexpr, - topk_stride1: tl.constexpr, - packed_stride0: tl.constexpr, - packed_stride1: tl.constexpr, - packed_stride2: tl.constexpr, - num_cols, - DCP_RANK: tl.constexpr, - DCP_WORLD_SIZE: tl.constexpr, - CP_INTERLEAVE: tl.constexpr, - HAS_ROW_STARTS: tl.constexpr, - TOPK: tl.constexpr, - BLOCK_SIZE: tl.constexpr, +class PackDCPTopkCandidatesKernel( + VllmTritonJitKernel["PackDCPTopkCandidatesKernel.CompileKey"] ): - row = tl.program_id(0) - tile = tl.program_id(1) - cols = tile * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) - mask = cols < TOPK + @dataclass(frozen=True) + class CompileKey: + dcp_rank: int + dcp_world_size: int + cp_interleave: int + has_row_starts: bool + topk: int + block_size: int - local_idx = tl.load( - topk_indices + row * topk_stride0 + cols * topk_stride1, - mask=mask, - other=-1, + # These scalars only describe runtime layouts and bounds; the constexpr + # fields below own the launch geometry and algorithmic specialization. + @staticmethod + @triton.jit( + do_not_specialize=[ + "logits_stride0", + "logits_stride1", + "topk_stride0", + "topk_stride1", + "packed_stride0", + "packed_stride1", + "packed_stride2", + "num_cols", + ] ) - valid = local_idx >= 0 - safe_local_idx = tl.maximum(local_idx, 0) + def kernel( + logits, + topk_indices, + packed, + row_starts, + logits_stride0, + logits_stride1, + topk_stride0, + topk_stride1, + packed_stride0, + packed_stride1, + packed_stride2, + num_cols, + DCP_RANK: tl.constexpr, + DCP_WORLD_SIZE: tl.constexpr, + CP_INTERLEAVE: tl.constexpr, + HAS_ROW_STARTS: tl.constexpr, + TOPK: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + ): + row = tl.program_id(0) + tile = tl.program_id(1) + cols = tile * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask = cols < TOPK - row_start = 0 - if HAS_ROW_STARTS: - row_start = tl.load(row_starts + row) + local_idx = tl.load( + topk_indices + row * topk_stride0 + cols * topk_stride1, + mask=mask, + other=-1, + ) + valid = local_idx >= 0 + safe_local_idx = tl.maximum(local_idx, 0) - score_col = safe_local_idx + row_start - score_col = tl.minimum(score_col, tl.maximum(num_cols - 1, 0)) - score = tl.load( - logits + row * logits_stride0 + score_col * logits_stride1, - mask=mask & valid, - other=-float("inf"), - ) + row_start = 0 + if HAS_ROW_STARTS: + row_start = tl.load(row_starts + row) - global_id = ( - (safe_local_idx // CP_INTERLEAVE) * (DCP_WORLD_SIZE * CP_INTERLEAVE) - + DCP_RANK * CP_INTERLEAVE - + safe_local_idx % CP_INTERLEAVE - ) - global_id = tl.where(valid, global_id, -1) + score_col = safe_local_idx + row_start + score_col = tl.minimum(score_col, tl.maximum(num_cols - 1, 0)) + score = tl.load( + logits + row * logits_stride0 + score_col * logits_stride1, + mask=mask & valid, + other=-float("inf"), + ) - packed_base = packed + row * packed_stride0 + cols * packed_stride1 - tl.store(packed_base, score, mask=mask) - tl.store(packed_base + packed_stride2, global_id.to(tl.float32), mask=mask) + global_id = ( + (safe_local_idx // CP_INTERLEAVE) * (DCP_WORLD_SIZE * CP_INTERLEAVE) + + DCP_RANK * CP_INTERLEAVE + + safe_local_idx % CP_INTERLEAVE + ) + global_id = tl.where(valid, global_id, -1) + + packed_base = packed + row * packed_stride0 + cols * packed_stride1 + tl.store(packed_base, score, mask=mask) + tl.store(packed_base + packed_stride2, global_id.to(tl.float32), mask=mask) + + def dispatch( # type: ignore[override] + self, + *, + has_row_starts: bool, + **compile_key_fields: int, + ) -> CompileKey: + return self.CompileKey( + **compile_key_fields, + has_row_starts=has_row_starts, + ) + + def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]: + dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size + if dcp_world_size <= 1: + return [] + cp_interleave = vllm_config.parallel_config.cp_kv_cache_interleave_size + topk = vllm_config.model_config.hf_config.index_topk + if topk <= 0: + return [] + + try: + from vllm.distributed.parallel_state import get_dcp_group + + dcp_rank = get_dcp_group().rank_in_group + except Exception: + dcp_rank = 0 + + return self._trace_dispatch(self.dispatch)( + dcp_rank=dcp_rank, + dcp_world_size=dcp_world_size, + cp_interleave=cp_interleave, + has_row_starts=(False, True), + topk=topk, + block_size=512, + ) + + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + fp32_ptr = TritonWarmupTensor(torch.float32) + int32_ptr = TritonWarmupTensor(torch.int32) + return dict( + logits=fp32_ptr, + topk_indices=TritonWarmupTensor(torch.int32, shape=(1, compile_key.topk)), + packed=fp32_ptr, + row_starts_arg=int32_ptr, + logits_stride0=1, + logits_stride1=1, + topk_stride0=1, + topk_stride1=1, + packed_stride0=1, + packed_stride1=1, + packed_stride2=1, + num_cols=1, + dcp_rank=compile_key.dcp_rank, + dcp_world_size=compile_key.dcp_world_size, + cp_interleave=compile_key.cp_interleave, + has_row_starts=compile_key.has_row_starts, + topk=compile_key.topk, + block_size=compile_key.block_size, + ) + + @kernel_launcher + def __call__( + self, + logits: torch.Tensor, + topk_indices: torch.Tensor, + packed: torch.Tensor, + row_starts_arg: torch.Tensor, + *, + logits_stride0: int, + logits_stride1: int, + topk_stride0: int, + topk_stride1: int, + packed_stride0: int, + packed_stride1: int, + packed_stride2: int, + num_cols: int, + dcp_rank: int, + dcp_world_size: int, + cp_interleave: int, + has_row_starts: bool, + topk: int, + block_size: int, + ) -> LaunchSpec: + grid = (topk_indices.shape[0], triton.cdiv(topk, block_size)) + return grid, dict( + row_starts=row_starts_arg, + DCP_RANK=dcp_rank, + DCP_WORLD_SIZE=dcp_world_size, + CP_INTERLEAVE=cp_interleave, + HAS_ROW_STARTS=has_row_starts, + TOPK=topk, + BLOCK_SIZE=block_size, + num_warps=8, + ) -@cute.jit -def _warp_scan_inclusive_i32(val: Int32, lane: Int32) -> Int32: - for i in cutlass.range_constexpr(cute.arch.WARP_SIZE.bit_length() - 1): - offset = 1 << i - partial = cute.arch.shuffle_sync_up(val, offset=offset, mask_and_clamp=0) - if lane >= offset: - val += partial - return val - - -@cute.jit -def _block_scan_inclusive_i32( - val: Int32, - lane: Int32, - warp_id: Int32, - warp_scratch: cute.Tensor, - warps_per_block: int, -) -> Int32: - prefix = _warp_scan_inclusive_i32(val, lane) - if lane == Int32(cute.arch.WARP_SIZE - 1): - warp_scratch[0, warp_id] = prefix - cute.arch.sync_threads() - - if warp_id == Int32(0): - warp_total = Int32(0) - if lane < Int32(warps_per_block): - warp_total = warp_scratch[0, lane] - warp_prefix = _warp_scan_inclusive_i32(warp_total, lane) - if lane < Int32(warps_per_block): - warp_scratch[0, lane] = warp_prefix - warp_total - cute.arch.sync_threads() - - return prefix + warp_scratch[0, warp_id] - - -class StableTopKFromGatheredCandidatesKernel: +class StableTopKFromGatheredCandidatesKernel( + VllmCuTeDSLJitKernel["StableTopKFromGatheredCandidatesKernel.CompileKey"] +): tb_size = 512 hist_bins = 2048 radix_bits = (hist_bins - 1).bit_length() - assert hist_bins == 1 << radix_bits key_bits = Uint64.width radix_passes = (key_bits + radix_bits - 1) // radix_bits final_radix_bits = key_bits - radix_bits * (radix_passes - 1) hist_chunks = (hist_bins + tb_size - 1) // tb_size warps_per_block = tb_size // cute.arch.WARP_SIZE - def __init__(self, topk: int, num_candidates: int): - assert num_candidates % self.tb_size == 0, ( + @dataclass(frozen=True) + class CompileKey: + topk: int + num_candidates: int + + @staticmethod + def kernel(compile_key: CompileKey) -> Any: + tb_size = StableTopKFromGatheredCandidatesKernel.tb_size + hist_bins = StableTopKFromGatheredCandidatesKernel.hist_bins + radix_bits = StableTopKFromGatheredCandidatesKernel.radix_bits + key_bits = StableTopKFromGatheredCandidatesKernel.key_bits + radix_passes = StableTopKFromGatheredCandidatesKernel.radix_passes + final_radix_bits = StableTopKFromGatheredCandidatesKernel.final_radix_bits + hist_chunks = StableTopKFromGatheredCandidatesKernel.hist_chunks + warps_per_block = StableTopKFromGatheredCandidatesKernel.warps_per_block + topk = compile_key.topk + assert hist_bins == 1 << radix_bits + assert compile_key.num_candidates % tb_size == 0, ( "StableTopKFromGatheredCandidatesKernel requires candidate count " - f"to be a multiple of {self.tb_size}, got {num_candidates}" + f"to be a multiple of {tb_size}, got {compile_key.num_candidates}" ) - self.topk = topk - self.keys_per_thread = num_candidates // self.tb_size + keys_per_thread = compile_key.num_candidates // tb_size @cute.struct class SharedStorage: - hist: cute.struct.MemRange[Int32, self.hist_bins] + hist: cute.struct.MemRange[Int32, hist_bins] committed_count: cute.struct.MemRange[Int32, 1] running_count: cute.struct.MemRange[Int32, 1] threshold_bin: cute.struct.MemRange[Int32, 1] threshold_found: cute.struct.MemRange[Int32, 1] include_threshold_bin: cute.struct.MemRange[Int32, 1] prefix_s: cute.struct.Align[cute.struct.MemRange[Uint64, 1], 8] - warp_totals: cute.struct.MemRange[Int32, self.warps_per_block] + warp_totals: cute.struct.MemRange[Int32, warps_per_block] - self.shared_storage = SharedStorage + shared_storage = SharedStorage - @cute.jit - def __call__( - self, - gathered: cute.Tensor, - out: cute.Tensor, - stream: CUstream, - ): - grid = (gathered.shape[0], 1, 1) - self.kernel(gathered, out).launch( - grid=grid, - block=(self.tb_size, 1, 1), - stream=stream, - ) - - @cute.jit - def _stable_key(self, score: Float32, token_id: Int32) -> Uint64: - bits = recast_val(score, Uint32) - mask = Uint32(0x80000000) - if (bits & Uint32(0x80000000)) != Uint32(0): - mask = Uint32(0xFFFFFFFF) - score_key = Uint64(bits ^ mask) << Uint64(32) - id_key = Uint64(~Uint32(token_id)) - key = score_key | id_key - if token_id < Int32(0): - key = Uint64(0) - return key - - @cute.jit - def _prefix_matches( - self, - key: Uint64, - prefix: Uint64, - prefix_bits: Int32, - ): - matches = prefix_bits == Int32(0) - if prefix_bits != Int32(0): - shift = Int32(self.key_bits) - prefix_bits - matches = (key >> Uint64(shift)) == (prefix >> Uint64(shift)) - return matches - - @cute.jit - def _radix_pass( - self, - keys: cute.Tensor, - output: cute.Tensor, - storage, - tid: Int32, - step: Int32, - bits: int, - is_final_pass: bool, - ): - hist_smem = storage.hist.get_tensor(cute.make_layout((self.hist_bins,))) - committed_count_smem = storage.committed_count.data_ptr() - running_count_smem = storage.running_count.data_ptr() - threshold_bin_smem = storage.threshold_bin.data_ptr() - threshold_found_smem = storage.threshold_found.data_ptr() - include_threshold_bin_smem = storage.include_threshold_bin.data_ptr() - prefix_smem = storage.prefix_s.data_ptr() - warp_totals_smem = storage.warp_totals.get_tensor( - cute.make_layout((1, self.warps_per_block)) - ) - - prefix_bits = step * Int32(self.radix_bits) - num_bins = 1 << bits - block_scan_iterations = (num_bins + self.tb_size - 1) // self.tb_size - shift = Int32(self.key_bits) - prefix_bits - Int32(bits) - bin_mask = Uint64(num_bins - 1) - prefix = prefix_smem.load() - - for chunk in cutlass.range_constexpr(self.hist_chunks): - hist_smem[tid + Int32(chunk * self.tb_size)] = Int32(0) - if tid == Int32(0): - running_count_smem.store(committed_count_smem.load()) - include_threshold_bin_smem.store(Int32(0)) - threshold_found_smem.store(Int32(0)) - cute.arch.sync_threads() - - for key_idx in cutlass.range_constexpr(self.keys_per_thread): - key = keys[key_idx] - if self._prefix_matches(key, prefix, prefix_bits): - bin_idx = Int32((key >> Uint64(shift)) & bin_mask) - cute.arch.atomic_add( - hist_smem.iterator + bin_idx, - Int32(1), - sem="relaxed", - scope="cta", + @cute.jit + def warp_scan_inclusive_i32(val: Int32, lane: Int32) -> Int32: + for i in cutlass.range_constexpr(cute.arch.WARP_SIZE.bit_length() - 1): + offset = 1 << i + partial = cute.arch.shuffle_sync_up( + val, offset=offset, mask_and_clamp=0 ) - cute.arch.sync_threads() + if lane >= offset: + val += partial + return val - lane = cute.arch.lane_idx() - warp_id = cute.arch.warp_idx() - # Each iteration scans one tb_size-wide slice of bins, high to low. - iter = Int32(0) - threshold_found = threshold_found_smem.load() - while threshold_found == Int32(0) and iter < Int32(block_scan_iterations): - bin_idx = Int32(num_bins - 1) - (iter * Int32(self.tb_size) + tid) - count = hist_smem[bin_idx] - chunk_inclusive = _block_scan_inclusive_i32( - count, - lane, - warp_id, - warp_totals_smem, - self.warps_per_block, + @cute.jit + def block_scan_inclusive_i32( + val: Int32, + lane: Int32, + warp_id: Int32, + warp_scratch: cute.Tensor, + warps_per_block: int, + ) -> Int32: + prefix = warp_scan_inclusive_i32(val, lane) + if lane == Int32(cute.arch.WARP_SIZE - 1): + warp_scratch[0, warp_id] = prefix + cute.arch.sync_threads() + + if warp_id == Int32(0): + warp_total = Int32(0) + if lane < Int32(warps_per_block): + warp_total = warp_scratch[0, lane] + warp_prefix = warp_scan_inclusive_i32(warp_total, lane) + if lane < Int32(warps_per_block): + warp_scratch[0, lane] = warp_prefix - warp_total + cute.arch.sync_threads() + + return prefix + warp_scratch[0, warp_id] + + @cute.jit + def stable_key(score: Float32, token_id: Int32) -> Uint64: + bits = recast_val(score, Uint32) + mask = Uint32(0x80000000) + if (bits & Uint32(0x80000000)) != Uint32(0): + mask = Uint32(0xFFFFFFFF) + score_key = Uint64(bits ^ mask) << Uint64(32) + id_key = Uint64(~Uint32(token_id)) + key = score_key | id_key + if token_id < Int32(0): + key = Uint64(0) + return key + + @cute.jit + def prefix_matches( + key: Uint64, + prefix: Uint64, + prefix_bits: Int32, + ): + matches = prefix_bits == Int32(0) + if prefix_bits != Int32(0): + shift = Int32(key_bits) - prefix_bits + matches = (key >> Uint64(shift)) == (prefix >> Uint64(shift)) + return matches + + @cute.jit + def radix_pass( + keys: cute.Tensor, + output: cute.Tensor, + storage, + tid: Int32, + step: Int32, + bits: int, + is_final_pass: bool, + ): + hist_smem = storage.hist.get_tensor(cute.make_layout((hist_bins,))) + committed_count_smem = storage.committed_count.data_ptr() + running_count_smem = storage.running_count.data_ptr() + threshold_bin_smem = storage.threshold_bin.data_ptr() + threshold_found_smem = storage.threshold_found.data_ptr() + include_threshold_bin_smem = storage.include_threshold_bin.data_ptr() + prefix_smem = storage.prefix_s.data_ptr() + warp_totals_smem = storage.warp_totals.get_tensor( + cute.make_layout((1, warps_per_block)) ) - running_count = running_count_smem.load() - prior_in_scan_slice = chunk_inclusive - count - remaining = Int32(self.topk) - running_count - prior_in_scan_slice - if count > Int32(0) and remaining > Int32(0) and remaining <= count: - threshold_bin_smem.store(bin_idx) - if count <= remaining or cutlass.const_expr(is_final_pass): - include_threshold_bin_smem.store(Int32(1)) - threshold_found_smem.store(Int32(1)) - # Barrier: every thread must finish reading running_count for this - # slice before tb_size-1 advances it, else a warp racing ahead to - # the store makes a lagging thread double-count the slice total - # (-> remaining too small -> threshold too high -> under-fill). - cute.arch.sync_threads() - if tid == Int32(self.tb_size - 1): - running_count_smem.store(running_count + chunk_inclusive) + + prefix_bits = step * Int32(radix_bits) + num_bins = 1 << bits + block_scan_iterations = (num_bins + tb_size - 1) // tb_size + shift = Int32(key_bits) - prefix_bits - Int32(bits) + bin_mask = Uint64(num_bins - 1) + prefix = prefix_smem.load() + + for chunk in cutlass.range_constexpr(hist_chunks): + hist_smem[tid + Int32(chunk * tb_size)] = Int32(0) + if tid == Int32(0): + running_count_smem.store(committed_count_smem.load()) + include_threshold_bin_smem.store(Int32(0)) + threshold_found_smem.store(Int32(0)) cute.arch.sync_threads() - threshold_found = threshold_found_smem.load() - iter += Int32(1) - - threshold = threshold_bin_smem.load() - should_include_threshold = include_threshold_bin_smem.load() != Int32(0) - for key_idx in cutlass.range_constexpr(self.keys_per_thread): - key = keys[key_idx] - if self._prefix_matches(key, prefix, prefix_bits): - bin_idx = Int32((key >> Uint64(shift)) & bin_mask) - selected = bin_idx > threshold - if should_include_threshold: - selected = selected or bin_idx == threshold - if selected: - dst = cute.arch.atomic_add( - committed_count_smem, + for key_idx in cutlass.range_constexpr(keys_per_thread): + key = keys[key_idx] + if prefix_matches(key, prefix, prefix_bits): + bin_idx = Int32((key >> Uint64(shift)) & bin_mask) + cute.arch.atomic_add( + hist_smem.iterator + bin_idx, Int32(1), sem="relaxed", scope="cta", ) - if dst < Int32(self.topk): - output[dst] = recast_val(~Uint32(key), Int32) - cute.arch.sync_threads() + cute.arch.sync_threads() - pass_finished = include_threshold_bin_smem.load() - if tid == Int32(0) and pass_finished == Int32(0): - prefix_smem.store(prefix | (Uint64(threshold) << Uint64(shift))) - cute.arch.sync_threads() - return pass_finished + lane = cute.arch.lane_idx() + warp_id = cute.arch.warp_idx() + # Each iteration scans one tb_size-wide slice of bins, high to low. + iter = Int32(0) + threshold_found = threshold_found_smem.load() + while threshold_found == Int32(0) and iter < Int32(block_scan_iterations): + bin_idx = Int32(num_bins - 1) - (iter * Int32(tb_size) + tid) + count = hist_smem[bin_idx] + chunk_inclusive = block_scan_inclusive_i32( + count, + lane, + warp_id, + warp_totals_smem, + warps_per_block, + ) + running_count = running_count_smem.load() + prior_in_scan_slice = chunk_inclusive - count + remaining = Int32(topk) - running_count - prior_in_scan_slice + if count > Int32(0) and remaining > Int32(0) and remaining <= count: + threshold_bin_smem.store(bin_idx) + if count <= remaining or cutlass.const_expr(is_final_pass): + include_threshold_bin_smem.store(Int32(1)) + threshold_found_smem.store(Int32(1)) + # Barrier: every thread must finish reading running_count for this + # slice before tb_size-1 advances it, else a warp racing ahead to + # the store makes a lagging thread double-count the slice total + # (-> remaining too small -> threshold too high -> under-fill). + cute.arch.sync_threads() + if tid == Int32(tb_size - 1): + running_count_smem.store(running_count + chunk_inclusive) + cute.arch.sync_threads() - @cute.kernel - def kernel( + threshold_found = threshold_found_smem.load() + iter += Int32(1) + + threshold = threshold_bin_smem.load() + should_include_threshold = include_threshold_bin_smem.load() != Int32(0) + for key_idx in cutlass.range_constexpr(keys_per_thread): + key = keys[key_idx] + if prefix_matches(key, prefix, prefix_bits): + bin_idx = Int32((key >> Uint64(shift)) & bin_mask) + selected = bin_idx > threshold + if should_include_threshold: + selected = selected or bin_idx == threshold + if selected: + dst = cute.arch.atomic_add( + committed_count_smem, + Int32(1), + sem="relaxed", + scope="cta", + ) + if dst < Int32(topk): + output[dst] = recast_val(~Uint32(key), Int32) + cute.arch.sync_threads() + + pass_finished = include_threshold_bin_smem.load() + if tid == Int32(0) and pass_finished == Int32(0): + prefix_smem.store(prefix | (Uint64(threshold) << Uint64(shift))) + cute.arch.sync_threads() + return pass_finished + + @cute.kernel + def device_kernel(input: cute.Tensor, out: cute.Tensor): + row, _, _ = cute.arch.block_idx() + tid, _, _ = cute.arch.thread_idx() + input_row = input[row, None, None] + output_row = out[row, None] + keys = cute.make_rmem_tensor((keys_per_thread,), Uint64) + + smem = cutlass.utils.SmemAllocator() + storage = smem.allocate(shared_storage, 8) + committed_count_smem = storage.committed_count.data_ptr() + prefix_smem = storage.prefix_s.data_ptr() + for i in range(tid, topk, tb_size): + output_row[i] = Int32(-1) + + for key_idx in cutlass.range_constexpr(keys_per_thread): + col = tid + Int32(key_idx * tb_size) + score = Float32(input_row[col, 0]) + token_id = Int32(input_row[col, 1]) + keys[key_idx] = stable_key(score, token_id) + + if tid == Int32(0): + committed_count_smem.store(Int32(0)) + prefix_smem.store(Uint64(0)) + cute.arch.sync_threads() + + step = Int32(0) + finished = Int32(0) + while finished == Int32(0) and step < Int32(radix_passes - 1): + finished = radix_pass( + keys, + output_row, + storage, + tid, + step, + radix_bits, + False, + ) + step += Int32(1) + + if finished == Int32(0): + radix_pass( + keys, + output_row, + storage, + tid, + Int32(radix_passes - 1), + final_radix_bits, + True, + ) + + @cute.jit + def host_entrypoint( + gathered: cute.Tensor, + out: cute.Tensor, + stream: CUstream, + ): + grid = (gathered.shape[0], 1, 1) + device_kernel(gathered, out).launch( + grid=grid, + block=(tb_size, 1, 1), + stream=stream, + ) + + return host_entrypoint + + def dispatch( # type: ignore[override] self, - input: cute.Tensor, - out: cute.Tensor, - ): - row, _, _ = cute.arch.block_idx() - tid, _, _ = cute.arch.thread_idx() - input_row = input[row, None, None] - output_row = out[row, None] - keys = cute.make_rmem_tensor((self.keys_per_thread,), Uint64) + **compile_key_fields: int, + ) -> CompileKey: + return self.CompileKey(**compile_key_fields) - smem = cutlass.utils.SmemAllocator() - storage = smem.allocate(self.shared_storage, 8) - committed_count_smem = storage.committed_count.data_ptr() - prefix_smem = storage.prefix_s.data_ptr() - for i in range(tid, self.topk, self.tb_size): - output_row[i] = Int32(-1) + def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]: + dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size + if dcp_world_size <= 1: + return [] + topk = vllm_config.model_config.hf_config.index_topk + if topk <= 0: + return [] + return self._trace_dispatch(self.dispatch)( + topk=topk, + num_candidates=topk * dcp_world_size, + ) - for key_idx in cutlass.range_constexpr(self.keys_per_thread): - col = tid + Int32(key_idx * self.tb_size) - score = Float32(input_row[col, 0]) - token_id = Int32(input_row[col, 1]) - keys[key_idx] = self._stable_key(score, token_id) - - if tid == Int32(0): - committed_count_smem.store(Int32(0)) - prefix_smem.store(Uint64(0)) - cute.arch.sync_threads() - - step = Int32(0) - finished = Int32(0) - while finished == Int32(0) and step < Int32(self.radix_passes - 1): - finished = self._radix_pass( - keys, - output_row, - storage, - tid, - step, - self.radix_bits, - False, - ) - step += Int32(1) - - if finished == Int32(0): - self._radix_pass( - keys, - output_row, - storage, - tid, - Int32(self.radix_passes - 1), - self.final_radix_bits, - True, - ) - - @cache - @staticmethod - def compile(topk: int, num_candidates: int): + def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]: num_rows = cute.sym_int() - gathered = cute.runtime.make_fake_tensor( Float32, - (num_rows, num_candidates, 2), + (num_rows, compile_key.num_candidates, 2), stride=(cute.sym_int64(divisibility=2), 2, 1), assumed_align=8, ) - out = make_fake_tensor(Int32, (num_rows, topk), divisibility=1) - - kernel = StableTopKFromGatheredCandidatesKernel(topk, num_candidates) - stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True) - return cute.compile( - kernel, - gathered, - out, - stream, - options="--enable-tvm-ffi", + out = make_fake_tensor( + Int32, + (num_rows, compile_key.topk), + divisibility=1, ) + return gathered, out + + @cutedsl_kernel_launcher + def __call__( + self, + gathered: torch.Tensor, + out: torch.Tensor, + *, + topk: int, + ) -> CuTeDSLLaunchSpec[CompileKey]: + compile_key = self.dispatch(topk=topk, num_candidates=gathered.shape[1]) + return ( + compile_key, + (gathered, out), + { + "gathered_shape": tuple(gathered.shape), + "out_shape": tuple(out.shape), + "topk": topk, + }, + ) + + +_PACK_DCP_TOPK_CANDIDATES_KERNEL = PackDCPTopkCandidatesKernel() +_STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL = StableTopKFromGatheredCandidatesKernel() diff --git a/vllm/model_executor/layers/sparse_attn_indexer.py b/vllm/model_executor/layers/sparse_attn_indexer.py index bb3572b178..488eb3899c 100644 --- a/vllm/model_executor/layers/sparse_attn_indexer.py +++ b/vllm/model_executor/layers/sparse_attn_indexer.py @@ -768,7 +768,8 @@ class SparseAttnIndexer(CustomOp): # DCP scalars are constant for the run; resolve them here (config is set # during model construction) and pass them into the custom op, rather # than threading them through per-step metadata. - parallel_config = get_current_vllm_config().parallel_config + vllm_config = get_current_vllm_config() + parallel_config = vllm_config.parallel_config self._parallel_config = parallel_config self.dcp_world_size = parallel_config.decode_context_parallel_size self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0 @@ -780,6 +781,28 @@ class SparseAttnIndexer(CustomOp): "the current vLLM environment." ) + if vllm_config.kernel_config.enable_jit_warmup: + from vllm.v1.attention.ops.common import ( + _PACK_SEQ_TRITON_KERNEL, + _UNPACK_SEQ_TRITON_KERNEL, + ) + + pack_dtype = torch.uint8 if use_fp4_cache else current_platform.fp8_dtype() + _PACK_SEQ_TRITON_KERNEL.register_warmup( + dtype=pack_dtype, + pad_value=0 if use_fp4_cache else -float("inf"), + ) + _UNPACK_SEQ_TRITON_KERNEL.register_warmup() + + if self.dcp_world_size > 1 and current_platform.is_cuda() and has_cutedsl(): + from vllm.model_executor.kernels.attention.dsa.dcp_indexer_cutedsl import ( # noqa: E501 + _PACK_DCP_TOPK_CANDIDATES_KERNEL, + _STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL, + ) + + _PACK_DCP_TOPK_CANDIDATES_KERNEL.register_warmup() + _STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL.register_warmup() + @property def cp_kv_cache_interleave_size(self) -> int: """With PD+DCP, the real value isn't known until block_size is finalized, diff --git a/vllm/model_executor/warmup/jit_warmup.py b/vllm/model_executor/warmup/jit_warmup.py index 99c387a3d1..d3216da111 100644 --- a/vllm/model_executor/warmup/jit_warmup.py +++ b/vllm/model_executor/warmup/jit_warmup.py @@ -670,6 +670,34 @@ class VllmJitKernel(Generic[CompileKeyT], ABC): self.compile(compile_key) +def _same_value(left: Any, right: Any) -> bool: + """True if two registration values are the same object or compare equal. + + Identity is checked first so shared singletons (e.g. ``vllm_config``, torch + dtypes) short-circuit before any potentially deep or non-boolean ``__eq__``. + """ + if left is right: + return True + try: + return bool(left == right) + except (TypeError, ValueError, RuntimeError): + return False + + +def _same_registration( + left: tuple[tuple[Any, ...], dict[str, Any]], + right: tuple[tuple[Any, ...], dict[str, Any]], +) -> bool: + """True if two ``(args, kwargs)`` registrations are equivalent.""" + left_args, left_kwargs = left + right_args, right_kwargs = right + if len(left_args) != len(right_args) or left_kwargs.keys() != right_kwargs.keys(): + return False + return all( + _same_value(x, y) for x, y in zip(left_args, right_args, strict=True) + ) and all(_same_value(left_kwargs[k], right_kwargs[k]) for k in left_kwargs) + + class JitWarmupRegistry: """Collect and compile JIT kernels selected during runner setup.""" @@ -713,13 +741,15 @@ class JitWarmupRegistry: kwargs: dict[str, Any], ) -> None: registrations = self._registrations.setdefault(kernel, []) - if ( - not args - and not kwargs - and any( - not registered_args and not registered_kwargs - for registered_args, registered_kwargs in registrations - ) + # Every layer of a deep model appends an identical (args, kwargs) here + # (e.g. a 61-layer DSA model registers the same pack (dtype, pad_value) + # 61 times); each expands to the same compile keys, so tracing + # get_warmup_keys once per distinct registration is sufficient. Dedup on + # identity-or-equality -- identity short-circuits shared singletons like + # vllm_config before any deep __eq__. + if any( + _same_registration(registered, (args, kwargs)) + for registered in registrations ): return registrations.append((args, kwargs)) diff --git a/vllm/model_executor/warmup/jit_warmup_cutedsl_helper.py b/vllm/model_executor/warmup/jit_warmup_cutedsl_helper.py new file mode 100644 index 0000000000..7311cc18a4 --- /dev/null +++ b/vllm/model_executor/warmup/jit_warmup_cutedsl_helper.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from abc import abstractmethod +from collections.abc import Callable, Mapping +from functools import wraps +from typing import Any, ClassVar, Generic, TypeAlias, TypeVar + +from vllm.model_executor.warmup.jit_warmup import VllmJitKernel + +DEFAULT_CUTEDSL_COMPILE_OPTIONS = "--enable-tvm-ffi" + +CompileKeyT = TypeVar("CompileKeyT") +CuTeDSLLaunchSpec: TypeAlias = tuple[ + CompileKeyT, + tuple[Any, ...], + Mapping[str, Any] | None, +] + + +def cutedsl_fake_stream(*, use_tvm_ffi_env_stream: bool = True) -> Any: + import cutlass.cute as cute + + return cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=use_tvm_ffi_env_stream) + + +def compile_cutedsl( + entry: Callable[..., Any], + *args: Any, + options: str = DEFAULT_CUTEDSL_COMPILE_OPTIONS, + use_tvm_ffi_env_stream: bool = True, +) -> Any: + import cutlass.cute as cute + + return cute.compile( + entry, + *args, + cutedsl_fake_stream(use_tvm_ffi_env_stream=use_tvm_ffi_env_stream), + options=options, + ) + + +class VllmCuTeDSLJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]): + """CuTeDSL owner whose compiled executor is shared by warmup and runtime.""" + + kernel: ClassVar[Any] + + @abstractmethod + def warmup_inputs(self, compile_key: CompileKeyT) -> tuple[Any, ...]: + """Return fake arguments that compile one executor specialization.""" + raise NotImplementedError + + def compile(self, compile_key: CompileKeyT) -> None: + if compile_key in self._compiled_cache: + return + self._compiled_cache[compile_key] = compile_cutedsl( + self.kernel(compile_key), + *self.warmup_inputs(compile_key), + ) + + +def cutedsl_kernel_launcher( + call_fn: Callable[..., CuTeDSLLaunchSpec[CompileKeyT]], +) -> Callable[..., Any]: + """Invoke a cached CuTeDSL executor from a declarative ``__call__``.""" + + @wraps(call_fn) + def wrapper( + self: VllmCuTeDSLJitKernel[CompileKeyT], + *args: Any, + **kwargs: Any, + ) -> Any: + compile_key, launch_args, runtime_context = call_fn(self, *args, **kwargs) + executor = self._get_or_compile( + compile_key, + runtime_context=runtime_context, + ) + return executor(*launch_args) + + return wrapper diff --git a/vllm/model_executor/warmup/jit_warmup_triton_helper.py b/vllm/model_executor/warmup/jit_warmup_triton_helper.py index 3a87f33599..b9b991ef82 100644 --- a/vllm/model_executor/warmup/jit_warmup_triton_helper.py +++ b/vllm/model_executor/warmup/jit_warmup_triton_helper.py @@ -106,8 +106,16 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]): self._warming = False @cached_property - def _kernel_param_names(self) -> frozenset[str]: - return frozenset(self.kernel.arg_names) + def _kernel_arg_names(self) -> tuple[str, ...]: + arg_names = getattr(self.kernel, "arg_names", None) + if arg_names is not None: + return tuple(arg_names) + wrapped = getattr(self.kernel, "func", None) + if wrapped is not None: + return tuple(inspect.signature(wrapped).parameters) + raise TypeError( + f"Cannot inspect kernel parameters for {type(self.kernel).__name__}" + ) def launch( self, @@ -116,14 +124,22 @@ class VllmTritonJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]): /, **kwargs: Any, ) -> Any: + runtime_launcher = kwargs.pop("_runtime_launcher", None) + runtime_launcher_arg_count = kwargs.pop("_runtime_launcher_arg_count", 0) for name, value in inputs.items(): - target = name if name in self._kernel_param_names else f"{name}_ptr" - if target in self._kernel_param_names and target not in kwargs: + target = name if name in self._kernel_arg_names else f"{name}_ptr" + if target in self._kernel_arg_names and target not in kwargs: kwargs[target] = value if self._warming: warmup = getattr(self.kernel, "warmup", None) assert warmup is not None return warmup(grid=(1,), **kwargs) + if runtime_launcher is not None: + regular_args = [ + kwargs.pop(name) + for name in self._kernel_arg_names[:runtime_launcher_arg_count] + ] + return runtime_launcher(self.kernel, grid, *regular_args, **kwargs) return self.kernel[grid](**kwargs) diff --git a/vllm/v1/attention/ops/common.py b/vllm/v1/attention/ops/common.py index f3316ac557..88ce371226 100644 --- a/vllm/v1/attention/ops/common.py +++ b/vllm/v1/attention/ops/common.py @@ -1,63 +1,157 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from dataclasses import dataclass +from typing import Any + import torch +from vllm.model_executor.warmup.jit_warmup_triton_helper import ( + LaunchSpec, + TritonWarmupTensor, + VllmTritonJitKernel, + kernel_launcher, +) from vllm.triton_utils import tl, triton -@triton.jit -def _pack_seq_kernel( - x_ptr, # [N, D] - out_ptr, # [B, Lmax, D] - lengths_ptr, # *i32, [B] - N: tl.constexpr, - D: tl.constexpr, - Lmax: tl.constexpr, - PAD_VALUE: tl.constexpr, - PAD_IS_UINT8: tl.constexpr, - BLOCK_T: tl.constexpr, # timesteps per program - BLOCK_D: tl.constexpr, # features per program -): - pid_b = tl.program_id(0) # batch id - pid_t = tl.program_id(1) # block over time dimension - pid_d = tl.program_id(2) # block over feature dimension - off_t = pid_t * BLOCK_T + tl.arange(0, BLOCK_T) # [BLOCK_T] - off_d = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) # [BLOCK_D] +class PackSeqTritonKernel(VllmTritonJitKernel["PackSeqTritonKernel.CompileKey"]): + @dataclass(frozen=True) + class CompileKey: + dtype: torch.dtype + pad_value: float | int + pad_is_uint8: bool + block_t: int + block_d: int - # Compute start index and sequence length from cumulative lengths - in_start = 0 - for i in range(pid_b): - in_start += tl.load(lengths_ptr + i) - seq_len = tl.load(lengths_ptr + pid_b) + # N is unused; D and Lmax only affect runtime addressing and masks. The + # compile-time tile shape remains controlled by BLOCK_T and BLOCK_D. + @staticmethod + @triton.jit(do_not_specialize=["N", "D", "Lmax"]) + def kernel( + x_ptr, # [N, D] + out_ptr, # [B, Lmax, D] + lengths_ptr, # *i32, [B] + N, + D, + Lmax, + PAD_VALUE: tl.constexpr, + PAD_IS_UINT8: tl.constexpr, + BLOCK_T: tl.constexpr, # timesteps per program + BLOCK_D: tl.constexpr, # features per program + ): + pid_b = tl.program_id(0) # batch id + pid_t = tl.program_id(1) # block over time dimension + pid_d = tl.program_id(2) # block over feature dimension + off_t = pid_t * BLOCK_T + tl.arange(0, BLOCK_T) # [BLOCK_T] + off_d = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) # [BLOCK_D] - # valid time positions for this block - t_mask = off_t < Lmax + # Compute start index and sequence length from cumulative lengths + in_start = 0 + for i in range(pid_b): + in_start += tl.load(lengths_ptr + i) + seq_len = tl.load(lengths_ptr + pid_b) - # compute input row indices for valid (b, t) - in_row = in_start + off_t - valid_row = (off_t < seq_len) & t_mask + # valid time positions for this block + t_mask = off_t < Lmax - # Pointers - # x_ptr: row-major [N, D] - x_row_ptr = x_ptr + in_row[:, None] * D + off_d[None, :] + # compute input row indices for valid (b, t) + in_row = in_start + off_t + valid_row = (off_t < seq_len) & t_mask - # out_ptr: row-major [B, Lmax, D] - out_row_ptr = out_ptr + (pid_b * Lmax + off_t)[:, None] * D + off_d[None, :] + # Pointers + # x_ptr: row-major [N, D] + x_row_ptr = x_ptr + in_row[:, None] * D + off_d[None, :] - # Initialize with PAD. PAD_IS_UINT8 selects the pad tensor's dtype so - # integer-typed outputs (e.g. MXFP4 packed nibbles, ue8m0 scale bytes) - # get an exact-byte pad rather than going through an fp32→uint8 cast - # that's implementation-defined outside of value 0. - d_mask = off_d[None, :] < D - if PAD_IS_UINT8: - pad_vals = tl.full([BLOCK_T, BLOCK_D], PAD_VALUE, tl.uint8) - else: - pad_vals = tl.full([BLOCK_T, BLOCK_D], PAD_VALUE, tl.float32) - tl.store(out_row_ptr, pad_vals, mask=t_mask[:, None] & d_mask) + # out_ptr: row-major [B, Lmax, D] + out_row_ptr = out_ptr + (pid_b * Lmax + off_t)[:, None] * D + off_d[None, :] - # Load & write only where within seq_len - x_vals = tl.load(x_row_ptr, mask=valid_row[:, None] & d_mask) - tl.store(out_row_ptr, x_vals, mask=valid_row[:, None] & d_mask) + # Initialize with PAD. PAD_IS_UINT8 selects the pad tensor's dtype so + # integer-typed outputs (e.g. MXFP4 packed nibbles, ue8m0 scale bytes) + # get an exact-byte pad rather than going through an fp32→uint8 cast + # that's implementation-defined outside of value 0. + d_mask = off_d[None, :] < D + if PAD_IS_UINT8: + pad_vals = tl.full([BLOCK_T, BLOCK_D], PAD_VALUE, tl.uint8) + else: + pad_vals = tl.full([BLOCK_T, BLOCK_D], PAD_VALUE, tl.float32) + tl.store(out_row_ptr, pad_vals, mask=t_mask[:, None] & d_mask) + + # Load & write only where within seq_len + x_vals = tl.load(x_row_ptr, mask=valid_row[:, None] & d_mask) + tl.store(out_row_ptr, x_vals, mask=valid_row[:, None] & d_mask) + + def dispatch( # type: ignore[override] + self, + *, + dtype: torch.dtype, + pad_value: float | int, + **compile_key_fields: int, + ) -> CompileKey: + is_uint8 = dtype == torch.uint8 + return self.CompileKey( + **compile_key_fields, + dtype=dtype, + pad_value=int(pad_value) if is_uint8 else float(pad_value), + pad_is_uint8=is_uint8, + ) + + def get_warmup_keys( + self, + *, + dtype: torch.dtype, + pad_value: float | int, + ) -> list[CompileKey]: + return self._trace_dispatch(self.dispatch)( + dtype=dtype, + pad_value=pad_value, + block_t=64, + block_d=64, + ) + + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + return dict( + x_reshaped=TritonWarmupTensor(compile_key.dtype), + out=TritonWarmupTensor(compile_key.dtype), + lengths=TritonWarmupTensor(torch.int32), + N=1, + D=1, + Lmax=1, + pad_value=compile_key.pad_value, + pad_is_uint8=compile_key.pad_is_uint8, + block_t=compile_key.block_t, + block_d=compile_key.block_d, + ) + + @kernel_launcher + def __call__( + self, + x_reshaped: torch.Tensor, + out: torch.Tensor, + lengths: torch.Tensor, + *, + N: int, + D: int, + Lmax: int, + pad_value: float | int, + pad_is_uint8: bool, + block_t: int, + block_d: int, + ) -> LaunchSpec: + grid = ( + lengths.shape[0], + triton.cdiv(Lmax, block_t), + triton.cdiv(D, block_d), + ) + return grid, dict( + x_ptr=x_reshaped, + lengths_ptr=lengths, + PAD_VALUE=pad_value, + PAD_IS_UINT8=pad_is_uint8, + BLOCK_T=block_t, + BLOCK_D=block_d, + num_warps=4, + num_stages=2, + ) def pack_seq_triton( @@ -109,20 +203,18 @@ def pack_seq_triton( out = torch.empty((B, Lmax, D), device=x.device, dtype=x.dtype) - grid = (B, triton.cdiv(Lmax, block_t), triton.cdiv(D, block_d)) - _pack_seq_kernel[grid]( + lengths = lengths.int() + _PACK_SEQ_TRITON_KERNEL( x_reshaped, out, - lengths.int(), - N, - D, - Lmax, - PAD_VALUE=pad_constexpr, - PAD_IS_UINT8=is_uint8, - BLOCK_T=block_t, - BLOCK_D=block_d, - num_warps=4, - num_stages=2, + lengths, + N=N, + D=D, + Lmax=Lmax, + pad_value=pad_constexpr, + pad_is_uint8=is_uint8, + block_t=block_t, + block_d=block_d, ) if len(original_shape) > 2: @@ -131,47 +223,109 @@ def pack_seq_triton( return out -@triton.jit -def _unpack_seq_triton_kernel( - packed_ptr, # [B, Lmax, D] - out_ptr, # [N, D] - lengths_ptr, # *i32, [B] - B: tl.constexpr, - Lmax: tl.constexpr, - D: tl.constexpr, - BLOCK_T: tl.constexpr, # timesteps per program - BLOCK_D: tl.constexpr, # features per program -): - pid_b = tl.program_id(0) # batch id - pid_t = tl.program_id(1) # block over time dimension - pid_d = tl.program_id(2) # block over feature dimension - off_t = pid_t * BLOCK_T + tl.arange(0, BLOCK_T) # [BLOCK_T] - off_d = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) # [BLOCK_D] +class UnpackSeqTritonKernel(VllmTritonJitKernel["UnpackSeqTritonKernel.CompileKey"]): + @dataclass(frozen=True) + class CompileKey: + dtype: torch.dtype + block_t: int + block_d: int - # bounds: compute start from cumulative lengths - in_start = 0 - for i in range(pid_b): - in_start += tl.load(lengths_ptr + i) - seq_len = tl.load(lengths_ptr + pid_b) + # B, Lmax, and D only affect runtime addressing and masks. The compile-time + # tile shape remains controlled by BLOCK_T and BLOCK_D. + @staticmethod + @triton.jit(do_not_specialize=["B", "Lmax", "D"]) + def kernel( + packed_ptr, # [B, Lmax, D] + out_ptr, # [N, D] + lengths_ptr, # *i32, [B] + B, + Lmax, + D, + BLOCK_T: tl.constexpr, # timesteps per program + BLOCK_D: tl.constexpr, # features per program + ): + pid_b = tl.program_id(0) # batch id + pid_t = tl.program_id(1) # block over time dimension + pid_d = tl.program_id(2) # block over feature dimension + off_t = pid_t * BLOCK_T + tl.arange(0, BLOCK_T) # [BLOCK_T] + off_d = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) # [BLOCK_D] - # valid time positions for this block - t_mask = off_t < Lmax - valid_row = (off_t < seq_len) & t_mask + # bounds: compute start from cumulative lengths + in_start = 0 + for i in range(pid_b): + in_start += tl.load(lengths_ptr + i) + seq_len = tl.load(lengths_ptr + pid_b) - # compute output row indices for valid (b, t) - out_row = in_start + off_t + # valid time positions for this block + t_mask = off_t < Lmax + valid_row = (off_t < seq_len) & t_mask - # Pointers - # packed_ptr: row-major [B, Lmax, D] - packed_row_ptr = packed_ptr + (pid_b * Lmax + off_t)[:, None] * D + off_d[None, :] + # compute output row indices for valid (b, t) + out_row = in_start + off_t - # out_ptr: row-major [N, D] - out_row_ptr = out_ptr + out_row[:, None] * D + off_d[None, :] + # Pointers + # packed_ptr: row-major [B, Lmax, D] + packed_row_ptr = ( + packed_ptr + (pid_b * Lmax + off_t)[:, None] * D + off_d[None, :] + ) - # Load from packed tensor and store to output - d_mask = off_d[None, :] < D - packed_vals = tl.load(packed_row_ptr, mask=valid_row[:, None] & d_mask) - tl.store(out_row_ptr, packed_vals, mask=valid_row[:, None] & d_mask) + # out_ptr: row-major [N, D] + out_row_ptr = out_ptr + out_row[:, None] * D + off_d[None, :] + + # Load from packed tensor and store to output + d_mask = off_d[None, :] < D + packed_vals = tl.load(packed_row_ptr, mask=valid_row[:, None] & d_mask) + tl.store(out_row_ptr, packed_vals, mask=valid_row[:, None] & d_mask) + + def dispatch( # type: ignore[override] + self, + *, + dtype: torch.dtype, + **compile_key_fields: int, + ) -> CompileKey: + return self.CompileKey(**compile_key_fields, dtype=dtype) + + def get_warmup_keys(self) -> list[CompileKey]: + return self._trace_dispatch(self.dispatch)( + dtype=torch.int32, + block_t=64, + block_d=64, + ) + + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + return dict( + packed_reshaped=TritonWarmupTensor(compile_key.dtype), + out=TritonWarmupTensor(compile_key.dtype), + lengths=TritonWarmupTensor(torch.int32), + B=1, + Lmax=1, + D=1, + block_t=compile_key.block_t, + block_d=compile_key.block_d, + ) + + @kernel_launcher + def __call__( + self, + packed_reshaped: torch.Tensor, + out: torch.Tensor, + lengths: torch.Tensor, + *, + B: int, + Lmax: int, + D: int, + block_t: int, + block_d: int, + ) -> LaunchSpec: + grid = (B, triton.cdiv(Lmax, block_t), triton.cdiv(D, block_d)) + return grid, dict( + packed_ptr=packed_reshaped, + lengths_ptr=lengths, + BLOCK_T=block_t, + BLOCK_D=block_d, + num_warps=4, + num_stages=2, + ) def unpack_seq_triton( @@ -209,18 +363,16 @@ def unpack_seq_triton( out = torch.empty((N, D), device=packed_tensor.device, dtype=packed_tensor.dtype) - grid = (B, triton.cdiv(Lmax, block_t), triton.cdiv(D, block_d)) - _unpack_seq_triton_kernel[grid]( + lengths = lengths.int() + _UNPACK_SEQ_TRITON_KERNEL( packed_reshaped, out, - lengths.int(), - B, - Lmax, - D, - BLOCK_T=block_t, - BLOCK_D=block_d, - num_warps=4, - num_stages=2, + lengths, + B=B, + Lmax=Lmax, + D=D, + block_t=block_t, + block_d=block_d, ) # Reshape output back to original dimensions (except first dimension) @@ -229,3 +381,7 @@ def unpack_seq_triton( out = out.reshape(output_shape) return out + + +_PACK_SEQ_TRITON_KERNEL = PackSeqTritonKernel() +_UNPACK_SEQ_TRITON_KERNEL = UnpackSeqTritonKernel() diff --git a/vllm/v1/attention/ops/dcp.py b/vllm/v1/attention/ops/dcp.py index 89fd611136..96b25870b8 100644 --- a/vllm/v1/attention/ops/dcp.py +++ b/vllm/v1/attention/ops/dcp.py @@ -6,7 +6,8 @@ from __future__ import annotations import functools from collections.abc import Callable -from typing import TYPE_CHECKING, Protocol +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Protocol import torch import torch.distributed as dist @@ -15,6 +16,16 @@ import vllm.envs as envs from vllm.config import VllmConfig from vllm.distributed import get_dcp_group from vllm.logger import init_logger +from vllm.model_executor.warmup.jit_warmup import ( + WarmupIntRange, +) +from vllm.model_executor.warmup.jit_warmup_triton_helper import ( + LaunchSpec, + TritonWarmupTensor, + VllmTritonJitKernel, + kernel_launcher, + triton_scalar_specialization_rep, +) from vllm.triton_utils import tl, triton from vllm.v1.attention.ops.cp_common import ( DirectCPWorkspace, @@ -65,93 +76,254 @@ def mask_dcp_empty_shards_( # AG + RS/AR implementation -@triton.jit -def _correct_attn_cp_out_kernel( - outputs_ptr, - new_output_ptr, - lses_ptr, - vlse_ptr, - outputs_stride_B, - outputs_stride_H, - outputs_stride_D, - lses_stride_N, - lses_stride_B, - lses_stride_H, - lse_idx, - HEAD_DIM: tl.constexpr, - N_ROUNDED: tl.constexpr, - IS_BASE_E: tl.constexpr, -): - """ - Apply the all-gathered lses to correct each local rank's attention - output. we still need perform a cross-rank reduction to obtain the - final attention output. +class CorrectAttnCPOutKernel(VllmTritonJitKernel["CorrectAttnCPOutKernel.CompileKey"]): + @dataclass(frozen=True) + class CompileKey: + output_dtype: torch.dtype + lse_dtype: torch.dtype + outputs_stride_b: int + outputs_stride_h: int + outputs_stride_d: int + lses_stride_n: int + lses_stride_b: int + lses_stride_h: int + lse_idx: int + head_dim: int + n_rounded: int + is_base_e: bool - Args: - outputs_ptr (triton.PointerType): - Pointer to input tensor of shape [ B, H, D ] - lses_ptr (triton.PointerType): - Pointer to input tensor of shape [ N, B, H ] - new_output_ptr (triton.PointerType): - Pointer to output tensor of shape [ B, H, D ] - vlse_ptr (triton.PointerType): - Pointer to output tensor of shape [ B, H ] - """ - batch_idx = tl.program_id(axis=0).to(tl.int64) - head_idx = tl.program_id(axis=1).to(tl.int64) - d_offsets = tl.arange(0, HEAD_DIM) - num_n_offsets = tl.arange(0, N_ROUNDED) + @staticmethod + @triton.jit + def kernel( + outputs_ptr, + new_output_ptr, + lses_ptr, + vlse_ptr, + outputs_stride_B, + outputs_stride_H, + outputs_stride_D, + lses_stride_N, + lses_stride_B, + lses_stride_H, + lse_idx, + HEAD_DIM: tl.constexpr, + N_ROUNDED: tl.constexpr, + IS_BASE_E: tl.constexpr, + ): + """ + Apply the all-gathered lses to correct each local rank's attention + output. we still need perform a cross-rank reduction to obtain the + final attention output. - # shape = [N] - lse_offsets = ( - num_n_offsets * lses_stride_N - + batch_idx * lses_stride_B - + head_idx * lses_stride_H - ) + Args: + outputs_ptr (triton.PointerType): + Pointer to input tensor of shape [ B, H, D ] + lses_ptr (triton.PointerType): + Pointer to input tensor of shape [ N, B, H ] + new_output_ptr (triton.PointerType): + Pointer to output tensor of shape [ B, H, D ] + vlse_ptr (triton.PointerType): + Pointer to output tensor of shape [ B, H ] + """ + batch_idx = tl.program_id(axis=0).to(tl.int64) + head_idx = tl.program_id(axis=1).to(tl.int64) + d_offsets = tl.arange(0, HEAD_DIM) + num_n_offsets = tl.arange(0, N_ROUNDED) - # calc final lse - lse = tl.load(lses_ptr + lse_offsets).to(tl.float32) - lse = tl.where((lse != lse) | (lse == float("inf")), -float("inf"), lse) - lse_max = tl.max(lse, axis=0) - lse_max = tl.where(lse_max == -float("inf"), 0, lse_max) - lse -= lse_max - if IS_BASE_E: - lse_exp = tl.exp(lse) - lse_acc = tl.sum(lse_exp, axis=0) - lse = tl.log(lse_acc) - else: - lse_exp = tl.exp2(lse) - lse_acc = tl.sum(lse_exp, axis=0) - lse = tl.log2(lse_acc) - lse += lse_max + # shape = [N] + lse_offsets = ( + num_n_offsets * lses_stride_N + + batch_idx * lses_stride_B + + head_idx * lses_stride_H + ) - lse_offsets = batch_idx * lses_stride_B + head_idx * lses_stride_H - tl.store(vlse_ptr + lse_offsets, lse) + # calc final lse + lse = tl.load(lses_ptr + lse_offsets).to(tl.float32) + lse = tl.where((lse != lse) | (lse == float("inf")), -float("inf"), lse) + lse_max = tl.max(lse, axis=0) + lse_max = tl.where(lse_max == -float("inf"), 0, lse_max) + lse -= lse_max + if IS_BASE_E: + lse_exp = tl.exp(lse) + lse_acc = tl.sum(lse_exp, axis=0) + lse = tl.log(lse_acc) + else: + lse_exp = tl.exp2(lse) + lse_acc = tl.sum(lse_exp, axis=0) + lse = tl.log2(lse_acc) + lse += lse_max - # shape = [D] - output_offsets = ( - batch_idx * outputs_stride_B - + head_idx * outputs_stride_H - + d_offsets * outputs_stride_D - ) + lse_offsets = batch_idx * lses_stride_B + head_idx * lses_stride_H + tl.store(vlse_ptr + lse_offsets, lse) - # correct output - lse_offset = ( - lse_idx * lses_stride_N + batch_idx * lses_stride_B + head_idx * lses_stride_H - ) - lse_tmp = tl.load(lses_ptr + lse_offset).to(tl.float32) - lse_finally = lse_tmp - lse - lse_finally = tl.where( - (lse_finally != lse_finally) | (lse_finally == float("inf")), - -float("inf"), - lse_finally, - ) - factor = tl.exp(lse_finally) if IS_BASE_E else tl.exp2(lse_finally) - output = tl.load(outputs_ptr + output_offsets) - output = output * factor - output = tl.where(factor == 0.0, 0.0, output) + # shape = [D] + output_offsets = ( + batch_idx * outputs_stride_B + + head_idx * outputs_stride_H + + d_offsets * outputs_stride_D + ) - tl.store(new_output_ptr + output_offsets, output) + # correct output + lse_offset = ( + lse_idx * lses_stride_N + + batch_idx * lses_stride_B + + head_idx * lses_stride_H + ) + lse_tmp = tl.load(lses_ptr + lse_offset).to(tl.float32) + lse_finally = lse_tmp - lse + lse_finally = tl.where( + (lse_finally != lse_finally) | (lse_finally == float("inf")), + -float("inf"), + lse_finally, + ) + factor = tl.exp(lse_finally) if IS_BASE_E else tl.exp2(lse_finally) + output = tl.load(outputs_ptr + output_offsets) + output = output * factor + output = tl.where(factor == 0.0, 0.0, output) + + tl.store(new_output_ptr + output_offsets, output) + + def dispatch( # type: ignore[override] + self, + *, + output_dtype: torch.dtype, + lse_dtype: torch.dtype, + num_tokens: int, + num_heads: int, + head_dim: int, + n_rounded: int, + lse_idx: int, + is_base_e: bool, + ) -> CompileKey: + # Warm the contiguous [B, H, D] output and [N, B, H] LSE layouts used + # by the production collective path. Runtime still forwards real + # strides so unsupported layouts remain correct, but may compile on + # first use under the JIT monitor. + outputs_stride_d = 1 + outputs_stride_h = head_dim + outputs_stride_b = num_heads * head_dim + lses_stride_h = 1 + lses_stride_b = num_heads + lses_stride_n = num_tokens * num_heads + return self.CompileKey( + output_dtype=output_dtype, + lse_dtype=lse_dtype, + outputs_stride_b=triton_scalar_specialization_rep(outputs_stride_b), + outputs_stride_h=triton_scalar_specialization_rep(outputs_stride_h), + outputs_stride_d=triton_scalar_specialization_rep(outputs_stride_d), + lses_stride_n=triton_scalar_specialization_rep(lses_stride_n), + lses_stride_b=triton_scalar_specialization_rep(lses_stride_b), + lses_stride_h=triton_scalar_specialization_rep(lses_stride_h), + lse_idx=triton_scalar_specialization_rep(lse_idx), + head_dim=head_dim, + n_rounded=n_rounded, + is_base_e=is_base_e, + ) + + def get_warmup_keys( + self, + vllm_config: Any, + *, + output_dtype: torch.dtype | None = None, + num_heads: int | None = None, + head_dim: int | None = None, + is_base_e: bool | tuple[bool, ...] = (False, True), + ) -> list[CompileKey]: + from vllm.model_executor.layers.attention.mla_attention import ( + get_mla_dims, + ) + + dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size + max_tokens = min( + 16, + vllm_config.scheduler_config.max_num_batched_tokens, + ) + hf_config = vllm_config.model_config.hf_config + if num_heads is None: + num_heads = ( + hf_config.num_attention_heads + // vllm_config.parallel_config.tensor_parallel_size + ) + if head_dim is None: + head_dim = get_mla_dims(vllm_config.model_config).v_head_dim + if output_dtype is None: + output_dtype = vllm_config.model_config.dtype + if dcp_world_size <= 1 or max_tokens <= 0 or num_heads <= 0: + return [] + return self._trace_dispatch(self.dispatch)( + output_dtype=output_dtype, + lse_dtype=torch.float32, + num_tokens=WarmupIntRange(1, max_tokens + 1), + num_heads=num_heads, + head_dim=head_dim, + n_rounded=dcp_world_size, + lse_idx=WarmupIntRange(0, dcp_world_size), + is_base_e=is_base_e, + ) + + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: + output_ptr = TritonWarmupTensor( + compile_key.output_dtype, + shape=(1, 1, compile_key.head_dim), + strides=( + compile_key.outputs_stride_b, + compile_key.outputs_stride_h, + compile_key.outputs_stride_d, + ), + ) + lse_ptr = TritonWarmupTensor( + compile_key.lse_dtype, + shape=(compile_key.n_rounded, 1, 1), + strides=( + compile_key.lses_stride_n, + compile_key.lses_stride_b, + compile_key.lses_stride_h, + ), + ) + return dict( + outputs=output_ptr, + new_output=output_ptr, + lses=lse_ptr, + vlse=lse_ptr, + lse_idx=compile_key.lse_idx, + ctx=None, + is_base_e=compile_key.is_base_e, + ) + + @kernel_launcher + def __call__( + self, + outputs: torch.Tensor, + new_output: torch.Tensor, + lses: torch.Tensor, + vlse: torch.Tensor, + lse_idx: int, + ctx: Any, + *, + is_base_e: bool, + ) -> LaunchSpec: + num_tokens, num_heads, head_dim = outputs.shape + n_rounded = lses.shape[0] + outputs_stride_b, outputs_stride_h, outputs_stride_d = outputs.stride() + lses_stride_n, lses_stride_b, lses_stride_h = lses.stride() + grid = (num_tokens, num_heads, 1) + return grid, dict( + outputs_stride_B=outputs_stride_b, + outputs_stride_H=outputs_stride_h, + outputs_stride_D=outputs_stride_d, + lses_stride_N=lses_stride_n, + lses_stride_B=lses_stride_b, + lses_stride_H=lses_stride_h, + HEAD_DIM=head_dim, + N_ROUNDED=n_rounded, + IS_BASE_E=is_base_e, + _runtime_launcher=None if self._warming else ctx.call_kernel, + # CPTritonContext caches the non-constexpr positional prefix; derive + # its length so adding/reordering a kernel arg cannot silently + # misalign the cached replay path. + _runtime_launcher_arg_count=len(self.kernel.arg_names) + - len(self.kernel.constexprs), + ) class CPTritonContext: @@ -202,14 +374,12 @@ def correct_attn_out( f"got {tuple(lses.shape)}" ) - B, H, D = out.shape - N = lses.shape[0] + B, H, _ = out.shape # Strides after we normalized shapes to 3-D views. The kernel computes # offsets for `vlse_ptr` using lses_stride_B/H, so the output buffer must # have the same B/H stride layout as a slice of `lses`. - o_sB, o_sH, o_sD = out.stride() - l_sN, l_sB, l_sH = lses.stride() + _, l_sB, l_sH = lses.stride() # Allocate LSE with the same B/H strides as `lses` so writes land correctly # even when `lses` is a non-contiguous view (e.g., 4-D to 3-D squeeze). @@ -217,24 +387,15 @@ def correct_attn_out( (B, H), (l_sB, l_sH), device=lses.device, dtype=lses.dtype ) - # Kernel launch config - grid = (B, H, 1) - - regular_args = ( + _CORRECT_ATTN_CP_OUT_KERNEL( out, out, lses, lse, - o_sB, - o_sH, - o_sD, - l_sN, - l_sB, - l_sH, cp_rank, + ctx, + is_base_e=is_lse_base_on_e, ) - const_args = {"HEAD_DIM": D, "N_ROUNDED": N, "IS_BASE_E": is_lse_base_on_e} - ctx.call_kernel(_correct_attn_cp_out_kernel, grid, *regular_args, **const_args) return out, lse @@ -1250,6 +1411,14 @@ class MLADCPManager: is_lse_base_on_e, use_pcp, ) + if vllm_config.kernel_config.enable_jit_warmup and not self.use_a2a: + _CORRECT_ATTN_CP_OUT_KERNEL.register_warmup( + vllm_config, + output_dtype=output_dtype, + num_heads=num_heads, + head_dim=output_head_dim, + is_base_e=is_lse_base_on_e, + ) self.query_gather = ( None if use_pcp @@ -1411,3 +1580,6 @@ class MLADCPManager: local_kv: torch.Tensor, ) -> object: return self._kv_gather(gathered_kv, local_kv) + + +_CORRECT_ATTN_CP_OUT_KERNEL = CorrectAttnCPOutKernel() diff --git a/vllm/v1/worker/block_table.py b/vllm/v1/worker/block_table.py index 5bf5df3b39..e8e6f80853 100644 --- a/vllm/v1/worker/block_table.py +++ b/vllm/v1/worker/block_table.py @@ -11,11 +11,11 @@ import torch from vllm.distributed import get_dcp_group, get_pcp_group from vllm.logger import init_logger -from vllm.model_executor.warmup.jit_warmup import ( - VllmJitKernel, -) from vllm.model_executor.warmup.jit_warmup_triton_helper import ( + LaunchSpec, TritonWarmupTensor, + VllmTritonJitKernel, + kernel_launcher, triton_scalar_specialization_rep, ) from vllm.triton_utils import tl, triton @@ -394,7 +394,9 @@ class MultiGroupBlockTable: return self.block_tables[idx] -class ComputeSlotMappingKernel(VllmJitKernel["ComputeSlotMappingKernel.CompileKey"]): +class ComputeSlotMappingKernel( + VllmTritonJitKernel["ComputeSlotMappingKernel.CompileKey"] +): triton_block_size = 1024 @dataclass(frozen=True) @@ -490,37 +492,53 @@ class ComputeSlotMappingKernel(VllmJitKernel["ComputeSlotMappingKernel.CompileKe def get_warmup_keys(self, **dispatch_kwargs: int) -> list[CompileKey]: return self._trace_dispatch(self.dispatch)(**dispatch_kwargs) - def compile(self, compile_key: CompileKey) -> None: - warmup = getattr(self.kernel, "warmup", None) - assert warmup is not None + def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]: int32_ptr = TritonWarmupTensor(torch.int32) int64_ptr = TritonWarmupTensor(torch.int64) - warmup( - 2, # arbitrary, num_tokens in do_not_specialize - 2, # arbitrary, max_num_tokens in do_not_specialize - int32_ptr, - int64_ptr, - int32_ptr, - compile_key.block_table_stride, - compile_key.block_size, - int64_ptr, - KV_CACHE_BLOCK_SIZE=compile_key.kv_cache_block_size, - BLOCKS_PER_KV_BLOCK=compile_key.blocks_per_kv_block, - TOTAL_CP_WORLD_SIZE=compile_key.total_cp_world_size, - TOTAL_CP_RANK=compile_key.total_cp_rank, - CP_KV_CACHE_INTERLEAVE_SIZE=compile_key.cp_kv_cache_interleave_size, - PAD_ID=PAD_SLOT_ID, - BLOCK_SIZE=self.triton_block_size, - grid=(2,), + return dict( + num_reqs=1, + num_tokens=2, # arbitrary, in do_not_specialize + max_num_tokens=2, # arbitrary, in do_not_specialize + query_start_loc=int32_ptr, + positions=int64_ptr, + block_table=TritonWarmupTensor( + torch.int32, + shape=(1, compile_key.block_table_stride), + ), + block_table_stride=compile_key.block_table_stride, + block_size=compile_key.block_size, + slot_mapping=int64_ptr, + kv_cache_block_size=compile_key.kv_cache_block_size, + blocks_per_kv_block=compile_key.blocks_per_kv_block, + total_cp_world_size=compile_key.total_cp_world_size, + total_cp_rank=compile_key.total_cp_rank, + cp_kv_cache_interleave_size=compile_key.cp_kv_cache_interleave_size, ) + @kernel_launcher def __call__( self, num_reqs: int, - *args: Any, - ) -> None: - self.kernel[(num_reqs + 1,)]( - *args, + num_tokens: int, + max_num_tokens: int, + query_start_loc: torch.Tensor, + positions: torch.Tensor, + block_table: torch.Tensor, + block_table_stride: int, + block_size: int, + slot_mapping: torch.Tensor, + kv_cache_block_size: int, + blocks_per_kv_block: int, + total_cp_world_size: int, + total_cp_rank: int, + cp_kv_cache_interleave_size: int, + ) -> LaunchSpec: + return (num_reqs + 1,), dict( + KV_CACHE_BLOCK_SIZE=kv_cache_block_size, + BLOCKS_PER_KV_BLOCK=blocks_per_kv_block, + TOTAL_CP_WORLD_SIZE=total_cp_world_size, + TOTAL_CP_RANK=total_cp_rank, + CP_KV_CACHE_INTERLEAVE_SIZE=cp_kv_cache_interleave_size, PAD_ID=PAD_SLOT_ID, BLOCK_SIZE=self.triton_block_size, )