diff --git a/csrc/libtorch_stable/cooperative_topk.cuh b/csrc/libtorch_stable/cooperative_topk.cuh index b43b9b8447..3312c127a7 100644 --- a/csrc/libtorch_stable/cooperative_topk.cuh +++ b/csrc/libtorch_stable/cooperative_topk.cuh @@ -484,7 +484,10 @@ __device__ void large_topk(const float* __restrict__ row_input, template __device__ void cooperative_topk_body(CooperativeTopKParams params) { const auto rank = blockIdx.y, row = blockIdx.x, tx = threadIdx.x; - const auto sl = params.lengths[row]; + // Clamp at 0: `sl` is compared signed here but cast to uint32 below, so a + // negative length would otherwise emit indices 0..TopK-1 as valid instead + // of the -1 padding. + const int32_t sl = params.lengths[row] > 0 ? params.lengths[row] : 0; int32_t* out = params.output + row * TopK; const float* in = params.input + row * params.stride; diff --git a/csrc/libtorch_stable/persistent_topk.cuh b/csrc/libtorch_stable/persistent_topk.cuh index 91717f53b2..9ae0fd4bcb 100644 --- a/csrc/libtorch_stable/persistent_topk.cuh +++ b/csrc/libtorch_stable/persistent_topk.cuh @@ -906,7 +906,26 @@ __global__ void __launch_bounds__(kThreadsPerBlock, 2) uint32_t row_idx = group_id + iter * num_groups; if (row_idx >= params.num_rows) break; - const uint32_t seq_len = params.lengths[row_idx]; + // Clamp the row length before any decision is made on it. + // + // `lengths` is int32 and is consumed here as uint32, so a negative value + // (e.g. a padded decode slot whose per-token context length underflowed) + // would reinterpret as ~4e9 and sail past every threshold below. Any + // value beyond the row width would also read into the next row. + // + // Clamping to max_seq_len additionally keeps this per-row decision + // consistent with the `cta_in_group != 0` early exit above, which is + // taken from the host-side scalar: when max_seq_len <= RADIX_THRESHOLD + // the non-leader CTAs return immediately, so a leader that reached the + // cooperative radix path would wait on the inter-CTA barrier for peers + // that no longer exist and spin until the kernel is killed. + const int32_t raw_len = params.lengths[row_idx]; + const uint32_t row_bound = + params.stride < params.max_seq_len ? params.stride : params.max_seq_len; + const uint32_t non_negative_len = + raw_len > 0 ? static_cast(raw_len) : 0u; + const uint32_t seq_len = + non_negative_len < row_bound ? non_negative_len : row_bound; int32_t* row_output = params.output + row_idx * params.top_k; const float* row_input = params.input + row_idx * params.stride; diff --git a/tests/kernels/attention/test_flashmla_sparse.py b/tests/kernels/attention/test_flashmla_sparse.py index 040ad50a5b..8082a359cc 100644 --- a/tests/kernels/attention/test_flashmla_sparse.py +++ b/tests/kernels/attention/test_flashmla_sparse.py @@ -225,6 +225,7 @@ def test_flashinfer_sparse_indices_cache(monkeypatch): token_to_req_indices=torch.tensor([0, 1, 1], dtype=torch.int32), decode_swa_indices=torch.tensor([[5, 6, -1, -1]], dtype=torch.int32), decode_swa_lens=torch.tensor([2], dtype=torch.int32), + decode_swa_width=4, is_valid_token=torch.tensor([True], dtype=torch.bool), num_decodes=1, num_prefills=1, @@ -331,3 +332,120 @@ def test_flashinfer_sparse_indices_cache(monkeypatch): assert builder_calls == 4 assert sparse_indices_third is not sparse_indices_fourth assert sparse_lens_third is not sparse_lens_fourth + + +def test_flashinfer_sparse_index_preserves_logical_window(monkeypatch): + from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as flashinfer_mod + from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata + + captured_shapes_and_windows: list[tuple[int, int]] = [] + + def fake_build(*args, **kwargs): + # window_size is the 12th positional arg of + # build_flashinfer_mixed_sparse_indices. + captured_shapes_and_windows.append((args[0].shape[-1], args[11])) + num_tokens = args[0].shape[0] + args[3].shape[0] + return ( + torch.zeros((num_tokens, 1), dtype=torch.int32), + torch.zeros((num_tokens,), dtype=torch.int32), + ) + + monkeypatch.setattr( + flashinfer_mod, "build_flashinfer_mixed_sparse_indices", fake_build + ) + + attn = object.__new__(flashinfer_mod.DeepseekV4FlashInferMLAAttention) + attn.compress_ratio = 1 + attn.window_size = 4 + attn.topk_indices_buffer = torch.zeros((4, 0), dtype=torch.int32) + + wide_width = 8 + wide_indices = torch.full((1, wide_width), -1, dtype=torch.int32) + wide_indices[0, :2] = torch.tensor([5, 6], dtype=torch.int32) + wide_metadata = DeepseekSparseSWAMetadata( + block_table=torch.tensor([[0, 1]], dtype=torch.int32), + slot_mapping=torch.tensor([0], dtype=torch.int64), + block_size=64, + seq_lens=torch.tensor([8], dtype=torch.int32), + query_start_loc=torch.tensor([0, 1], dtype=torch.int32), + query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.int32), + token_to_req_indices=torch.tensor([0], dtype=torch.int32), + decode_swa_indices=wide_indices, + decode_swa_lens=torch.tensor([2], dtype=torch.int32), + decode_swa_width=wide_width, + is_valid_token=torch.tensor([True], dtype=torch.bool), + num_decodes=1, + num_prefills=0, + num_decode_tokens=1, + num_prefill_tokens=0, + ) + attn._build_sparse_index_metadata( + kv_cache=None, + swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16), + swa_metadata=wide_metadata, + attn_metadata=None, + swa_only=True, + ) + assert captured_shapes_and_windows == [(wide_width, attn.window_size)] + + empty_width = 8 + empty_metadata = DeepseekSparseSWAMetadata( + block_table=torch.tensor([[0, 1]], dtype=torch.int32), + slot_mapping=torch.tensor([0, 1], dtype=torch.int64), + block_size=64, + seq_lens=torch.tensor([8], dtype=torch.int32), + query_start_loc=torch.tensor([0, 2], dtype=torch.int32), + query_start_loc_cpu=torch.tensor([0, 2], dtype=torch.int32), + token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32), + decode_swa_indices=torch.empty((0, 1, empty_width), dtype=torch.int32), + decode_swa_lens=torch.empty((0,), dtype=torch.int32), + decode_swa_width=empty_width, + is_valid_token=torch.tensor([True, True], dtype=torch.bool), + num_decodes=0, + num_prefills=1, + num_decode_tokens=0, + num_prefill_tokens=2, + ) + attn._build_sparse_index_metadata( + kv_cache=None, + swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16), + swa_metadata=empty_metadata, + attn_metadata=None, + swa_only=True, + ) + assert captured_shapes_and_windows == [ + (wide_width, attn.window_size), + (empty_width, attn.window_size), + ] + + +def test_flashinfer_mixed_sparse_indices_separates_window_and_padded_width(): + from vllm.models.deepseek_v4.common.ops.cache_utils import ( + build_flashinfer_mixed_sparse_indices, + ) + + device = torch.device("cuda") + padded_width = 8 + logical_window = 4 + sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices( + decode_swa_indices=torch.empty( + (0, padded_width), dtype=torch.int32, device=device + ), + decode_compressed_indices=None, + decode_compressed_topk_lens=None, + prefill_topk_indices=torch.empty((1, 0), dtype=torch.int32, device=device), + query_start_loc=torch.tensor([0, 1], dtype=torch.int32, device=device), + seq_lens=torch.tensor([logical_window], dtype=torch.int32, device=device), + token_to_req_indices=torch.tensor([0], dtype=torch.int32, device=device), + swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device), + swa_block_size=64, + compressed_block_table=None, + compressed_block_size=64, + window_size=logical_window, + compress_ratio=1, + topk=0, + ) + + assert sparse_indices.shape == (1, padded_width) + assert sparse_indices[0].cpu().tolist() == [0, 1, 2, 3, -1, -1, -1, -1] + assert sparse_lens.cpu().tolist() == [padded_width] diff --git a/tests/kernels/moe/test_ocp_mx_moe.py b/tests/kernels/moe/test_ocp_mx_moe.py index 9e74ec2fd3..5b8dfab25b 100644 --- a/tests/kernels/moe/test_ocp_mx_moe.py +++ b/tests/kernels/moe/test_ocp_mx_moe.py @@ -1095,6 +1095,101 @@ def test_convert_standard_mxfp4_weights_for_flashinfer_cutlass( torch.testing.assert_close(interleaved_scales[1], w2_scale) +def test_gpt_oss_quant_config_supplies_clamped_swiglu_params(): + from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend + from vllm.model_executor.layers.quantization.mxfp4 import GptOssMxfp4MoEMethod + + fake_method = types.SimpleNamespace( + mxfp4_backend=Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8, + w13_precision_config=None, + w2_precision_config=None, + ) + fake_layer = types.SimpleNamespace( + w13_weight_scale=torch.zeros(2, 4, 2, dtype=torch.uint8), + w2_weight_scale=torch.zeros(2, 4, 2, dtype=torch.uint8), + ) + + quant_config = GptOssMxfp4MoEMethod.get_fused_moe_quant_config( + fake_method, fake_layer + ) + + assert quant_config is not None + assert quant_config.gemm1_alpha == 1.702 + assert quant_config.gemm1_beta == 1.0 + assert quant_config.gemm1_clamp_limit == 7.0 + + +@pytest.mark.skipif( + not current_platform.is_cuda(), + reason="CUDA is required for FlashInferExperts parameter storage", +) +@pytest.mark.parametrize("supplied", [False, True]) +def test_flashinfer_experts_swiglu_params_follow_quant_config(supplied: bool): + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, + FusedMoEParallelConfig, + RoutingMethodType, + mxfp4_w4a16_moe_quant_config, + ) + from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import ( + FlashInferExperts, + ) + + num_experts, intermediate_size, hidden_size = 2, 128, 256 + w1_scale = torch.zeros( + (num_experts, 2 * intermediate_size, hidden_size // 32), + dtype=torch.uint8, + device="cuda", + ) + w2_scale = torch.zeros( + (num_experts, hidden_size, intermediate_size // 32), + dtype=torch.uint8, + device="cuda", + ) + extra = ( + {"gemm1_alpha": 0.5, "gemm1_beta": 0.25, "gemm1_clamp_limit": 3.0} + if supplied + else {} + ) + quant_config = mxfp4_w4a16_moe_quant_config( + w1_scale=w1_scale, w2_scale=w2_scale, **extra + ) + moe_config = FusedMoEConfig( + num_experts=num_experts, + experts_per_token=1, + hidden_dim=hidden_size, + intermediate_size=intermediate_size, + num_local_experts=num_experts, + num_logical_experts=num_experts, + activation=MoEActivation.SILU, + device="cuda", + routing_method=RoutingMethodType.TopK, + moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(), + in_dtype=torch.bfloat16, + ) + experts = FlashInferExperts(moe_config=moe_config, quant_config=quant_config) + + if supplied: + for name, value in ( + ("gemm1_alpha", 0.5), + ("gemm1_beta", 0.25), + ("gemm1_clamp_limit", 3.0), + ): + parameter = getattr(experts, name) + assert isinstance(parameter, torch.Tensor) + assert parameter.dtype == torch.float32 + assert parameter.shape == (num_experts,) + torch.testing.assert_close( + parameter.cpu(), + torch.full((num_experts,), value, dtype=torch.float32), + ) + else: + assert experts.gemm1_alpha is None + assert experts.gemm1_beta is None + assert experts.gemm1_clamp_limit is None + + @pytest.mark.parametrize("topk", [1, 4]) @pytest.mark.parametrize("num_experts", [32]) @pytest.mark.parametrize("num_tokens", [1, 128]) diff --git a/tests/kernels/test_compressor_kv_cache.py b/tests/kernels/test_compressor_kv_cache.py index d8878a0aea..c1d9a2106b 100644 --- a/tests/kernels/test_compressor_kv_cache.py +++ b/tests/kernels/test_compressor_kv_cache.py @@ -31,10 +31,23 @@ from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import ( ) from vllm.models.deepseek_v4.compressor import _get_c128_boundary from vllm.platforms import current_platform +from vllm.v1.attention.backends.mla.compressor_utils import ( + get_dspark_swa_index_width, +) from .test_fused_indexer_q_rope_quant import quantize_to_mxfp4 +@pytest.mark.parametrize( + ("window_size", "num_speculative_tokens", "expected"), + [(128, 5, 192), (512, 5, 576), (1024, 0, 1024)], +) +def test_get_dspark_swa_index_width( + window_size: int, num_speculative_tokens: int, expected: int +): + assert get_dspark_swa_index_width(window_size, num_speculative_tokens) == expected + + def test_compute_global_topk_reuses_output_buffers(): device = "cuda" topk_indices = torch.tensor( diff --git a/tests/v1/attention/test_flashinfer_sparse_mla_sm120_api.py b/tests/v1/attention/test_flashinfer_sparse_mla_sm120_api.py index 3a7677e751..12a8cf5cbc 100644 --- a/tests/v1/attention/test_flashinfer_sparse_mla_sm120_api.py +++ b/tests/v1/attention/test_flashinfer_sparse_mla_sm120_api.py @@ -7,6 +7,9 @@ from types import SimpleNamespace import torch from vllm.config import set_current_vllm_config +from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import ( + _required_sm120_sparse_topk, +) from vllm.platforms.interface import DeviceCapability from vllm.utils import flashinfer as fi_utils from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import ( @@ -52,3 +55,33 @@ def test_v32_glm_sm120_backend_accepts_glm_block_size( ) assert invalid_reasons == [] + + +def test_sm120_dsv4_capability_checks_exact_dispatch_shape(monkeypatch) -> None: + fake_module = SimpleNamespace( + _DECODE_DSV4_DISPATCH=frozenset({(32, 128), (32, 192)}) + ) + monkeypatch.setattr(fi_utils, "has_flashinfer_sparse_mla_sm120", lambda: True) + monkeypatch.setattr(fi_utils, "_get_submodule", lambda _name: fake_module) + fi_utils.has_flashinfer_sparse_mla_sm120_config.cache_clear() + + assert fi_utils.has_flashinfer_sparse_mla_sm120_config(32, 128) + assert fi_utils.has_flashinfer_sparse_mla_sm120_config(32, 192) + assert not fi_utils.has_flashinfer_sparse_mla_sm120_config(32, 256) + assert not fi_utils.has_flashinfer_sparse_mla_sm120_config(16, 192) + + fi_utils.has_flashinfer_sparse_mla_sm120_config.cache_clear() + + +def test_sm120_dsv4_required_topk_tracks_dspark_width() -> None: + causal = SimpleNamespace( + attention_config=SimpleNamespace(use_non_causal=False), + speculative_config=SimpleNamespace(num_speculative_tokens=5), + ) + dspark = SimpleNamespace( + attention_config=SimpleNamespace(use_non_causal=True), + speculative_config=SimpleNamespace(num_speculative_tokens=5), + ) + + assert _required_sm120_sparse_topk(causal, 128) == 128 + assert _required_sm120_sparse_topk(dspark, 128) == 192 diff --git a/tests/v1/spec_decode/test_dflash_prepare_inputs.py b/tests/v1/spec_decode/test_dflash_prepare_inputs.py new file mode 100644 index 0000000000..50a414a837 --- /dev/null +++ b/tests/v1/spec_decode/test_dflash_prepare_inputs.py @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import numpy as np +import pytest +import torch + +from vllm.v1.attention.backends.utils import PAD_SLOT_ID +from vllm.v1.worker.gpu.spec_decode.dflash.speculator import ( + prepare_dflash_inputs, +) + +pytestmark = pytest.mark.skipif( + not torch.cuda.is_available(), reason="requires a CUDA device" +) + + +def _run_prepare(*, target_positions: list[int], block_table_values: list[int]): + device = torch.device("cuda") + max_num_reqs = 4 + max_num_tokens = 16 + num_speculative_steps = 3 + + input_buffers = SimpleNamespace( + input_ids=torch.full((max_num_tokens,), -1, dtype=torch.int32, device=device), + positions=torch.full((max_num_tokens,), -1, dtype=torch.int64, device=device), + query_start_loc=torch.full( + (max_num_reqs + 1,), -1, dtype=torch.int32, device=device + ), + seq_lens=torch.full((max_num_reqs,), -1, dtype=torch.int32, device=device), + ) + input_batch = SimpleNamespace( + num_reqs=1, + num_scheduled_tokens=np.array([4], dtype=np.int32), + positions=torch.tensor(target_positions, dtype=torch.int64, device=device), + query_start_loc=torch.tensor([0, 4], dtype=torch.int32, device=device), + idx_mapping=torch.tensor([2], dtype=torch.int32, device=device), + ) + query_slot_mapping = torch.full( + (max_num_tokens,), -2, dtype=torch.int64, device=device + ) + context_positions = torch.full( + (max_num_tokens,), -1, dtype=torch.int64, device=device + ) + context_slot_mapping = torch.full( + (max_num_tokens,), -2, dtype=torch.int64, device=device + ) + sample_indices = torch.full( + (max_num_reqs * num_speculative_steps,), + -1, + dtype=torch.int64, + device=device, + ) + sample_pos = torch.full_like(sample_indices, -1) + sample_idx_mapping = torch.full( + sample_indices.shape, -1, dtype=torch.int32, device=device + ) + temperature = torch.zeros(max_num_reqs, dtype=torch.float32, device=device) + seeds = torch.zeros(max_num_reqs, dtype=torch.int64, device=device) + input_temperature = torch.tensor( + [0.0, 0.0, 1.0, 0.0], dtype=torch.float32, device=device + ) + input_seeds = torch.tensor([0, 0, 17, 0], dtype=torch.int64, device=device) + last_sampled = torch.tensor([0, 0, 99, 0], dtype=torch.int64, device=device) + next_prefill_tokens = torch.zeros_like(last_sampled) + block_table = torch.tensor([block_table_values], dtype=torch.int32, device=device) + + prepare_dflash_inputs( + input_buffers, + query_slot_mapping, + context_positions, + context_slot_mapping, + sample_indices, + sample_pos, + sample_idx_mapping, + temperature, + seeds, + input_batch, + torch.tensor([1], dtype=torch.int32, device=device), + torch.tensor([2], dtype=torch.int32, device=device), + last_sampled, + next_prefill_tokens, + input_temperature, + input_seeds, + block_table, + 4, + 123, + num_speculative_steps, + num_speculative_steps, + max_num_reqs, + max_num_tokens, + 128, + sample_from_anchor=True, + ) + torch.accelerator.synchronize() + return SimpleNamespace( + input_buffers=input_buffers, + query_slot_mapping=query_slot_mapping.cpu(), + context_positions=context_positions.cpu(), + context_slot_mapping=context_slot_mapping.cpu(), + sample_indices=sample_indices.cpu(), + sample_pos=sample_pos.cpu(), + sample_idx_mapping=sample_idx_mapping.cpu(), + temperature=temperature.cpu(), + seeds=seeds.cpu(), + ) + + +def test_prepare_dflash_inputs_excludes_rejected_context_suffix(): + # Positions 10/11 use physical block 7. Rejected positions 12/13 would use + # block 8, but must be PAD context rather than contaminating draft KV. + out = _run_prepare( + target_positions=[10, 11, 12, 13], + block_table_values=[0, 0, 7, 8, 9, 10, 11, 12], + ) + + assert out.context_positions[:4].tolist() == [10, 11, 0, 0] + assert out.context_slot_mapping[:4].tolist() == [30, 31, PAD_SLOT_ID, PAD_SLOT_ID] + + # The replacement query starts immediately after the two valid rows and + # advances from the last accepted position (11). + assert out.input_buffers.input_ids[:3].cpu().tolist() == [99, 123, 123] + assert out.input_buffers.positions[:3].cpu().tolist() == [12, 13, 14] + assert out.query_slot_mapping[:3].tolist() == [32, 33, 34] + assert out.sample_indices[:3].tolist() == [0, 1, 2] + assert out.sample_pos[:3].tolist() == [13, 14, 15] + assert out.sample_idx_mapping[:3].tolist() == [2, 2, 2] + assert out.temperature[2].item() == 1.0 + assert out.seeds[2].item() == 17 + + +def test_prepare_dflash_inputs_never_writes_the_null_block(): + # The valid context uses logical block 0 and the replacement query uses + # logical block 1. Both map to the null block and must remain unwritable. + out = _run_prepare( + target_positions=[2, 3, 4, 5], + block_table_values=[0, 0, 7, 8, 9, 10, 11, 12], + ) + + assert out.context_slot_mapping[:4].tolist() == [ + PAD_SLOT_ID, + PAD_SLOT_ID, + PAD_SLOT_ID, + PAD_SLOT_ID, + ] + assert out.query_slot_mapping[:3].tolist() == [ + PAD_SLOT_ID, + PAD_SLOT_ID, + PAD_SLOT_ID, + ] diff --git a/tests/v1/worker/test_workspace.py b/tests/v1/worker/test_workspace.py new file mode 100644 index 0000000000..a909ea9af8 --- /dev/null +++ b/tests/v1/worker/test_workspace.py @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import cast + +import pytest +import torch + +import vllm.v1.worker.workspace as workspace +from vllm.config import VllmConfig +from vllm.v1.worker.gpu_worker import _num_workspace_lanes + + +class _SpecConfig: + def __init__(self, dspark: bool) -> None: + self._dspark = dspark + + def use_dspark(self) -> bool: + return self._dspark + + +class _VllmConfig: + def __init__(self, spec_config: _SpecConfig | None) -> None: + self.speculative_config = spec_config + + +@pytest.mark.parametrize( + ("use_v2_model_runner", "spec_config", "expected"), + [ + (True, _SpecConfig(True), 2), + (False, _SpecConfig(True), 1), + (True, _SpecConfig(False), 1), + (True, None, 1), + ], +) +def test_workspace_lane_count_is_dspark_only( + use_v2_model_runner: bool, + spec_config: _SpecConfig | None, + expected: int, +) -> None: + config = cast(VllmConfig, _VllmConfig(spec_config)) + assert _num_workspace_lanes(config, use_v2_model_runner) == expected + + +def test_workspace_lanes_do_not_alias_and_restore_context(monkeypatch) -> None: + monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: 0) + manager = workspace.WorkspaceManager( + torch.device("cpu"), num_ubatches=2, num_lanes=2 + ) + + assert manager._current_workspaces == [None, None, None, None] + + (target,) = manager.get_simultaneous(((512,), torch.uint8)) + with workspace.use_workspace_lane(1): + (draft,) = manager.get_simultaneous(((256,), torch.uint8)) + (draft_reused,) = manager.get_simultaneous(((8,), torch.uint8)) + (target_reused,) = manager.get_simultaneous(((8,), torch.uint8)) + + assert manager._current_workspaces[0].numel() == 512 # type: ignore[union-attr] + assert manager._current_workspaces[1].numel() == 256 # type: ignore[union-attr] + assert manager._current_workspaces[2:] == [None, None] + assert target.data_ptr() != draft.data_ptr() + assert draft.data_ptr() == draft_reused.data_ptr() + assert target.data_ptr() == target_reused.data_ptr() + + +def test_workspace_lanes_compose_with_ubatches(monkeypatch) -> None: + active_ubatch = [0] + monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: active_ubatch[0]) + manager = workspace.WorkspaceManager( + torch.device("cpu"), num_ubatches=2, num_lanes=2 + ) + + pointers = set() + for ubatch_id in range(2): + active_ubatch[0] = ubatch_id + for lane in range(2): + with workspace.use_workspace_lane(lane): + (buffer,) = manager.get_simultaneous(((16,), torch.uint8)) + pointers.add(buffer.data_ptr()) + + assert len(pointers) == 4 + + +def test_workspace_lane_validation(monkeypatch) -> None: + monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: 0) + manager = workspace.WorkspaceManager(torch.device("cpu"), num_lanes=1) + + with ( + pytest.raises(ValueError, match="non-negative"), + workspace.use_workspace_lane(-1), + ): + pass + + with ( + workspace.use_workspace_lane(1), + pytest.raises(RuntimeError, match="is not configured"), + ): + manager.get_simultaneous(((1,), torch.uint8)) + + with pytest.raises(ValueError, match="at least one"): + workspace.WorkspaceManager(torch.device("cpu"), num_lanes=0) diff --git a/vllm/model_executor/layers/fused_moe/experts/flashinfer_cutlass_moe.py b/vllm/model_executor/layers/fused_moe/experts/flashinfer_cutlass_moe.py index 77afcf0d66..1614e41b33 100644 --- a/vllm/model_executor/layers/fused_moe/experts/flashinfer_cutlass_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/flashinfer_cutlass_moe.py @@ -111,21 +111,15 @@ class FlashInferExperts(mk.FusedMoEExpertsModular): self.gemm1_alpha = _per_expert(quant_config.gemm1_alpha) self.gemm1_beta = _per_expert(quant_config.gemm1_beta) - if quant_config.weight_quant_dtype == "mxfp4": - # This value is used specifically for gpt-oss, - # Need to revisit this for other models - if self.gemm1_alpha is None: - self.gemm1_alpha = _per_expert(1.702) - if self.gemm1_beta is None: - self.gemm1_beta = _per_expert(1.0) - if self.gemm1_clamp_limit is None: - self.gemm1_clamp_limit = _per_expert(7.0) - if quant_config.quant_dtype == "mxfp8": - self.fake_input_scale = torch.ones( - self.num_experts, - device=self.device, - dtype=torch.float32, - ) + if ( + quant_config.weight_quant_dtype == "mxfp4" + and quant_config.quant_dtype == "mxfp8" + ): + self.fake_input_scale = torch.ones( + self.num_experts, + device=self.device, + dtype=torch.float32, + ) @property def expects_unquantized_inputs(self) -> bool: @@ -324,9 +318,6 @@ class FlashInferExperts(mk.FusedMoEExpertsModular): elif self.weight_quant_dtype == "mxfp4": assert self.w1_scale is not None and self.w2_scale is not None assert w1.is_contiguous() and w2.is_contiguous() - assert self.gemm1_alpha is not None - assert self.gemm1_beta is not None - assert self.gemm1_clamp_limit is not None assert topk_ids.is_contiguous() fc1_expert_biases = self.w1_bias diff --git a/vllm/models/deepseek_v4/amd/rocm.py b/vllm/models/deepseek_v4/amd/rocm.py index 23223acd2d..6a457298dc 100644 --- a/vllm/models/deepseek_v4/amd/rocm.py +++ b/vllm/models/deepseek_v4/amd/rocm.py @@ -414,7 +414,9 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuild and base.decode_swa_lens is not None ): ragged_indices, ragged_indptr = build_ragged_indices_from_dense( - base.decode_swa_indices.reshape(base.num_decode_tokens, -1), + base.decode_swa_indices.reshape( + base.num_decode_tokens, base.decode_swa_width + ), base.decode_swa_lens, ) ragged_indices, ragged_indptr = _copy_ragged_to_graph_buffers( @@ -423,9 +425,7 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuild self.decode_swa_ragged_indices_buffer, self.decode_swa_ragged_indptr_buffer, base.num_decode_tokens, - # Actual dense width for this build: window_size (causal) or - # noncausal_index_width (DSpark non-causal draft). - base.decode_swa_indices.shape[-1], + base.decode_swa_width, ) return DeepseekV4ROCMAiterSparseSWAMetadata( diff --git a/vllm/models/deepseek_v4/common/ops/cache_utils.py b/vllm/models/deepseek_v4/common/ops/cache_utils.py index dc06dc91b2..811bf21d49 100644 --- a/vllm/models/deepseek_v4/common/ops/cache_utils.py +++ b/vllm/models/deepseek_v4/common/ops/cache_utils.py @@ -839,16 +839,17 @@ def build_flashinfer_mixed_sparse_indices( ) -> tuple[torch.Tensor, torch.Tensor]: """Build the FlashInfer DSV4 sparse-index matrix for decode-first batches. - Produces ``sparse_indices`` of shape ``[num_tokens, window_size + - padded_topk]`` (the first ``window_size`` columns are SWA slot ids, the rest - are compressed/top-k slot ids) and ``sparse_topk_lens`` (active length per - token). Decode tokens read precomputed SWA/compressed indices; prefill tokens - derive their SWA window from the position and translate local compressed - indices to global slots via the block tables. + Produces ``sparse_indices`` of shape ``[num_tokens, swa_index_width + + padded_topk]`` (the first ``swa_index_width`` columns are SWA slot ids, the + rest are compressed/top-k slot ids) and ``sparse_topk_lens`` (active length + per token). Decode tokens read precomputed SWA/compressed indices; prefill + tokens derive their SWA window from the position and translate local + compressed indices to global slots via the block tables. """ assert decode_swa_indices.dtype == torch.int32 assert decode_swa_indices.dim() == 2 - assert decode_swa_indices.shape[-1] == window_size + swa_index_width = decode_swa_indices.shape[-1] + assert swa_index_width >= window_size if decode_compressed_topk_lens is not None: assert decode_compressed_topk_lens.dtype == torch.int32 assert prefill_topk_indices.dtype == torch.int32 @@ -897,7 +898,7 @@ def build_flashinfer_mixed_sparse_indices( padded_topk = max(topk, decode_compressed_topk) padded_topk = (padded_topk + 3) // 4 * 4 sparse_indices = torch.empty( - (num_tokens, window_size + padded_topk), + (num_tokens, swa_index_width + padded_topk), dtype=torch.int32, device=decode_swa_indices.device, ) @@ -907,7 +908,7 @@ def build_flashinfer_mixed_sparse_indices( if num_tokens == 0: return sparse_indices, sparse_topk_lens - window_block_size = triton.next_power_of_2(max(window_size, 1)) + window_block_size = triton.next_power_of_2(max(swa_index_width, 1)) topk_block_size = triton.next_power_of_2(max(padded_topk, 1)) max_block_size = max(window_block_size, topk_block_size) num_warps = 4 if max_block_size >= 256 else 1 @@ -944,6 +945,7 @@ def build_flashinfer_mixed_sparse_indices( compressed_span, NUM_DECODE_TOKENS=num_decode_tokens, WINDOW_SIZE=window_size, + SWA_INDEX_WIDTH=swa_index_width, COMPRESS_RATIO=compress_ratio, TOP_K=topk, PADDED_TOP_K=padded_topk, @@ -1011,6 +1013,7 @@ def _build_flashinfer_mixed_sparse_indices_kernel( compressed_block_span, NUM_DECODE_TOKENS, WINDOW_SIZE: tl.constexpr, + SWA_INDEX_WIDTH: tl.constexpr, COMPRESS_RATIO: tl.constexpr, TOP_K: tl.constexpr, PADDED_TOP_K: tl.constexpr, @@ -1024,9 +1027,9 @@ def _build_flashinfer_mixed_sparse_indices_kernel( token_idx = tl.program_id(0) if token_idx < NUM_DECODE_TOKENS: - for i in range(0, WINDOW_SIZE, WINDOW_BLOCK_SIZE): + for i in range(0, SWA_INDEX_WIDTH, WINDOW_BLOCK_SIZE): offset = i + tl.arange(0, WINDOW_BLOCK_SIZE) - mask = offset < WINDOW_SIZE + mask = offset < SWA_INDEX_WIDTH values = tl.load( decode_swa_indices_ptr + token_idx * decode_swa_stride + offset, mask=mask, @@ -1072,7 +1075,7 @@ def _build_flashinfer_mixed_sparse_indices_kernel( tl.store( sparse_indices_ptr + token_idx * sparse_indices_stride - + WINDOW_SIZE + + SWA_INDEX_WIDTH + offset, values, mask=mask, @@ -1086,7 +1089,7 @@ def _build_flashinfer_mixed_sparse_indices_kernel( else: compressed_len = tl.full((), DECODE_COMPRESSED_TOPK, dtype=tl.int32) - tl.store(sparse_topk_lens_ptr + token_idx, WINDOW_SIZE + compressed_len) + tl.store(sparse_topk_lens_ptr + token_idx, SWA_INDEX_WIDTH + compressed_len) return prefill_idx = token_idx - NUM_DECODE_TOKENS @@ -1102,9 +1105,9 @@ def _build_flashinfer_mixed_sparse_indices_kernel( swa_start_pos = pos - swa_len + 1 topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K) - for i in range(0, WINDOW_SIZE, WINDOW_BLOCK_SIZE): + for i in range(0, SWA_INDEX_WIDTH, WINDOW_BLOCK_SIZE): offset = i + tl.arange(0, WINDOW_BLOCK_SIZE) - mask = offset < WINDOW_SIZE + mask = offset < SWA_INDEX_WIDTH pos_offset = swa_start_pos + offset block_indices = pos_offset // swa_block_size block_numbers = tl.load( @@ -1148,10 +1151,10 @@ def _build_flashinfer_mixed_sparse_indices_kernel( tl.store( sparse_indices_ptr + token_idx * sparse_indices_stride - + WINDOW_SIZE + + SWA_INDEX_WIDTH + offset, slot_ids, mask=mask, ) - tl.store(sparse_topk_lens_ptr + token_idx, WINDOW_SIZE + topk_len) + tl.store(sparse_topk_lens_ptr + token_idx, SWA_INDEX_WIDTH + topk_len) diff --git a/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py b/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py index 27031e02c5..e73156c1c1 100644 --- a/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py +++ b/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, ClassVar, cast import torch +from vllm.config import VllmConfig from vllm.config.cache import CacheDType from vllm.forward_context import get_forward_context from vllm.models.deepseek_v4.attention import DeepseekV4Attention @@ -25,6 +26,9 @@ from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability from vllm.utils.flashinfer import flashinfer_trtllm_batch_decode_sparse_mla_dsv4 from vllm.v1.attention.backend import MultipleOf +from vllm.v1.attention.backends.mla.compressor_utils import ( + get_dspark_swa_index_width, +) if TYPE_CHECKING: from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata @@ -74,6 +78,19 @@ def _pad_to_supported_q_heads(num_heads: int) -> int: ) +def _required_sm120_sparse_topk(vllm_config: VllmConfig, window_size: int) -> int: + """Return the SM120 DSV4 SWA specialization needed by this model.""" + if not vllm_config.attention_config.use_non_causal: + return window_size + speculative_config = vllm_config.speculative_config + if speculative_config is None: + return window_size + return get_dspark_swa_index_width( + window_size, + speculative_config.num_speculative_tokens, + ) + + class DeepseekV4FlashInferMLASparseBackend(DeepseekV4SparseMLABackend): """FlashInfer backend using the DSv4 sparse metadata/cache layout. @@ -309,7 +326,7 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention): assert swa_metadata.block_table is not None decode_swa_indices = swa_metadata.decode_swa_indices.reshape( - num_decode_tokens, self.window_size + num_decode_tokens, swa_metadata.decode_swa_width ) decode_compressed_topk_lens = None decode_compressed_indices_are_local = False @@ -568,14 +585,18 @@ class DeepseekV4FlashInferSM120Attention(DeepseekV4Attention): tma_aligned_scales=self._tma_aligned_scales, ) - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - from vllm.utils.flashinfer import has_flashinfer_sparse_mla_sm120 + def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None: + super().__init__(vllm_config, *args, **kwargs) + from vllm.utils.flashinfer import has_flashinfer_sparse_mla_sm120_config - if not has_flashinfer_sparse_mla_sm120(): + required_topk = _required_sm120_sparse_topk(vllm_config, self.window_size) + if not has_flashinfer_sparse_mla_sm120_config(self.padded_heads, required_topk): raise RuntimeError( - "FLASHINFER_MLA_SPARSE_DSV4 on SM120 requires FlashInfer's " - "sparse MLA decode API." + "FLASHINFER_MLA_SPARSE_DSV4 on SM120 requires a FlashInfer " + "DSV4 sparse MLA decode specialization for " + f"(num_q_heads={self.padded_heads}, top_k={required_topk}). " + "Install a FlashInfer build containing " + "flashinfer-ai/flashinfer#4380." ) self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe() # Per-tensor FP8 cache path scales. diff --git a/vllm/utils/flashinfer.py b/vllm/utils/flashinfer.py index ce92132a07..4f7d2eecd7 100644 --- a/vllm/utils/flashinfer.py +++ b/vllm/utils/flashinfer.py @@ -237,6 +237,22 @@ def has_flashinfer_sparse_mla_sm120() -> bool: ) +@functools.cache +def has_flashinfer_sparse_mla_sm120_config(num_q_heads: int, top_k: int) -> bool: + """Return whether FlashInfer ships an SM120 DSV4 decode specialization. + + The public sparse MLA API predates some DSV4 shapes, so checking only that + the callable exists can select a package that later aborts or rejects a + valid vLLM configuration. Inspect FlashInfer's dispatch table until it + exposes a public capability query. + """ + if not has_flashinfer_sparse_mla_sm120(): + return False + mod = _get_submodule("flashinfer.mla._sparse_mla_sm120") + dispatch = getattr(mod, "_DECODE_DSV4_DISPATCH", None) if mod else None + return dispatch is not None and (int(num_q_heads), int(top_k)) in dispatch + + @functools.cache def has_flashinfer_cutedsl() -> bool: """Return ``True`` if FlashInfer cutedsl module is available.""" diff --git a/vllm/v1/attention/backends/mla/compressor_utils.py b/vllm/v1/attention/backends/mla/compressor_utils.py index 36b115f644..47d331ba7b 100644 --- a/vllm/v1/attention/backends/mla/compressor_utils.py +++ b/vllm/v1/attention/backends/mla/compressor_utils.py @@ -3,6 +3,18 @@ import torch from vllm.triton_utils import tl, triton +from vllm.utils.math_utils import cdiv + +_DSPARK_SWA_INDEX_ALIGNMENT = 64 + + +def get_dspark_swa_index_width( + window_size: int, + num_speculative_tokens: int, +) -> int: + """Return the padded width of non-causal DSpark SWA indices.""" + width = max(int(window_size), 0) + max(int(num_speculative_tokens), 0) + return cdiv(width, _DSPARK_SWA_INDEX_ALIGNMENT) * _DSPARK_SWA_INDEX_ALIGNMENT @triton.jit diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index 717cfa621c..baecb391fd 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -57,9 +57,12 @@ def _prepare_uniform_decode_kernel( req_id = idx // max_decode_len local_idx = idx % max_decode_len - # Compute number of KVs attended to by this token. + # Compute number of KVs attended to by this token. Padding requests have + # seq_len == 0, which would otherwise make the first token of each padded + # request negative (e.g. next_n=2 gives 0-2+0+1 = -1). Downstream kernels + # read these as uint32, turning -1 into ~4e9. seq_len = tl.load(seq_lens_ptr + req_id) - per_token_seq_len = seq_len - max_decode_len + local_idx + 1 + per_token_seq_len = tl.maximum(seq_len - max_decode_len + local_idx + 1, 0) tl.store(decode_seq_lens_ptr + idx, per_token_seq_len) # Copy block table row. @@ -748,12 +751,15 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): seq_lens_buffer = self.decode_seq_lens_buffer[ : num_decodes * max_decode_len ].view(num_decodes, max_decode_len) + # Clamp at 0: padding requests have seq_len == 0, which would + # otherwise make token 0 negative (next_n=2 gives 0-2+1+0 = -1). + # Downstream kernels read these as uint32, turning -1 into ~4e9. seq_lens_buffer[:] = ( seq_lens.unsqueeze(1) - max_decode_len + 1 + self.offsets_buffer[:max_decode_len] - ) + ).clamp_(min=0) seq_lens = seq_lens_buffer return seq_lens, block_table, decode_lens, num_decodes, requires_padding diff --git a/vllm/v1/attention/backends/mla/sparse_swa.py b/vllm/v1/attention/backends/mla/sparse_swa.py index f4e5874f9d..d858eb677c 100644 --- a/vllm/v1/attention/backends/mla/sparse_swa.py +++ b/vllm/v1/attention/backends/mla/sparse_swa.py @@ -22,6 +22,9 @@ from vllm.v1.attention.backend import ( CommonAttentionMetadata, MultipleOf, ) +from vllm.v1.attention.backends.mla.compressor_utils import ( + get_dspark_swa_index_width, +) from vllm.v1.attention.backends.utils import split_decodes_and_prefills from vllm.v1.attention.ops.flashmla import FlashMLASchedMeta, get_mla_metadata from vllm.v1.kv_cache_interface import ( @@ -174,8 +177,10 @@ class DeepseekSparseSWAMetadata: is_valid_token: torch.Tensor | None = None # [num_tokens] token_to_req_indices: torch.Tensor | None = None # [num_tokens] - decode_swa_indices: torch.Tensor | None = None # [num_decode_tokens, window_size] + decode_swa_indices: torch.Tensor | None = None # [num_decode_tokens, width] decode_swa_lens: torch.Tensor | None = None # [num_decode_tokens] + # window_size (causal) or noncausal_index_width (DSpark non-causal). + decode_swa_width: int = 0 # Paged-coordinate prefill SWA indices/lens (FP8 paged-direct prefill). prefill_swa_indices: torch.Tensor | None = ( None # [num_prefill_tokens, 1, window_size] @@ -487,11 +492,14 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): # DSpark draft: the block is non-causal (every query attends to the # trailing window of context PLUS all query tokens, including future ones), # so its per-token index list is wider than `window_size`. The kernel pads - # the q-head count to B_TOPK (64/128), which requires the index width to be - # a multiple of 128. + # the q-head count to B_TOPK. Pad to a kernel-supported width; the logical + # SWA window remains unchanged when the padded matrix is built. self.is_dspark = spec_config is not None and spec_config.use_dspark() self.noncausal_index_width = ( - cdiv(self.window_size + self.num_speculative_tokens, 128) * 128 + get_dspark_swa_index_width( + self.window_size, + self.num_speculative_tokens, + ) if self.is_dspark else 0 ) @@ -536,6 +544,9 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): is_valid_token.copy_(slot_mapping >= 0) non_causal = not common_attn_metadata.causal + decode_swa_width = ( + self.noncausal_index_width if non_causal else self.window_size + ) decode_swa_indices = self.decode_swa_indices if num_decode_tokens > 0: self.decode_swa_lens[num_decode_tokens:] = 0 @@ -634,6 +645,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): token_to_req_indices=token_to_req_indices, decode_swa_indices=decode_swa_indices[:num_decode_tokens], decode_swa_lens=self.decode_swa_lens[:num_decode_tokens], + decode_swa_width=decode_swa_width, prefill_swa_indices=( self.prefill_swa_indices[:num_prefill_tokens] if num_prefill_tokens > 0 diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 93c3e250a2..c92ffedf59 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -149,6 +149,7 @@ from vllm.v1.worker.utils import ( copy_kv_cache_blocks_inplace, get_uniform_decode_token_count, ) +from vllm.v1.worker.workspace import use_workspace_lane logger = init_logger(__name__) @@ -164,6 +165,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.parallel_config = vllm_config.parallel_config self.scheduler_config = vllm_config.scheduler_config self.speculative_config = vllm_config.speculative_config + self._draft_workspace_lane = int( + self.speculative_config is not None and self.speculative_config.use_dspark() + ) self.observability_config = vllm_config.observability_config self.device = device @@ -364,10 +368,11 @@ class GPUModelRunner(LoRAModelRunnerMixin): assert self.speculative_config is not None set_eagle3_aux_hidden_state_layers(self.model, self.speculative_config) if isinstance(self.speculator, DraftModelSpeculator): - self.speculator.load_model(self.model) - eplb_models_added = self.eplb.maybe_register_speculator( - self.speculator, self.speculative_config, load_dummy_weights - ) + with use_workspace_lane(self._draft_workspace_lane): + self.speculator.load_model(self.model) + eplb_models_added = self.eplb.maybe_register_speculator( + self.speculator, self.speculative_config, load_dummy_weights + ) time_after_load = time.perf_counter() self.model_memory_usage = m.consumed_memory @@ -732,27 +737,28 @@ class GPUModelRunner(LoRAModelRunnerMixin): if hasattr(self.model, "get_mtp_target_hidden_states"): pre_hc_hidden_states = self.model.get_mtp_target_hidden_states() spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr] - self.speculator.propose( - input_batch=input_batch, - attn_metadata=attn_metadata, - slot_mappings=slot_mappings_by_layer, - last_hidden_states=spec_hidden_states, - aux_hidden_states=aux_hidden_states, - num_sampled=torch.ones( - input_batch.num_reqs, dtype=torch.int32, device=self.device - ), - num_rejected=torch.zeros( - input_batch.num_reqs, dtype=torch.int32, device=self.device - ), - last_sampled=self.req_states.last_sampled_tokens, - next_prefill_tokens=self.req_states.next_prefill_tokens, - temperature=self.sampler.sampling_states.temperature.gpu, - seeds=self.sampler.sampling_states.seeds.gpu, - dummy_run=True, - skip_attn_for_dummy_run=skip_attn, - mm_inputs=mm_inputs, - is_profile=is_profile, - ) + with use_workspace_lane(self._draft_workspace_lane): + self.speculator.propose( + input_batch=input_batch, + attn_metadata=attn_metadata, + slot_mappings=slot_mappings_by_layer, + last_hidden_states=spec_hidden_states, + aux_hidden_states=aux_hidden_states, + num_sampled=torch.ones( + input_batch.num_reqs, dtype=torch.int32, device=self.device + ), + num_rejected=torch.zeros( + input_batch.num_reqs, dtype=torch.int32, device=self.device + ), + last_sampled=self.req_states.last_sampled_tokens, + next_prefill_tokens=self.req_states.next_prefill_tokens, + temperature=self.sampler.sampling_states.temperature.gpu, + seeds=self.sampler.sampling_states.seeds.gpu, + dummy_run=True, + skip_attn_for_dummy_run=skip_attn, + mm_inputs=mm_inputs, + is_profile=is_profile, + ) self.step_timing.drafter_end() assert hidden_states is not None # Last PP rank always has hidden_states @@ -880,7 +886,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): lora_capture_hook=create_lora_capture_hook(self.lora_config, self), ) if self.speculator is not None: - self.speculator.capture() + with use_workspace_lane(self._draft_workspace_lane): + self.speculator.capture() if self.adaptive_verification is not None: with self.step_timing.collect() as timings: for batch in self.adaptive_verification.batches_to_profile( @@ -1799,20 +1806,21 @@ class GPUModelRunner(LoRAModelRunnerMixin): if hasattr(self.model, "get_mtp_target_hidden_states"): pre_hc_hidden_states = self.model.get_mtp_target_hidden_states() spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr] - draft_tokens = self.speculator.propose( - input_batch, - attn_metadata, - slot_mappings_by_layer, - spec_hidden_states, - aux_hidden_states, - num_sampled, - num_rejected, - self.req_states.last_sampled_tokens, - self.req_states.next_prefill_tokens, - self.sampler.sampling_states.temperature.gpu, - self.sampler.sampling_states.seeds.gpu, - mm_inputs=mm_inputs, - ) + with use_workspace_lane(self._draft_workspace_lane): + draft_tokens = self.speculator.propose( + input_batch, + attn_metadata, + slot_mappings_by_layer, + spec_hidden_states, + aux_hidden_states, + num_sampled, + num_rejected, + self.req_states.last_sampled_tokens, + self.req_states.next_prefill_tokens, + self.sampler.sampling_states.temperature.gpu, + self.sampler.sampling_states.seeds.gpu, + mm_inputs=mm_inputs, + ) self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens if self.adaptive_verification is not None: self.adaptive_verification.record_confidences( diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py index 97c284c03d..eb5dc470b2 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py @@ -84,8 +84,11 @@ class DFlashSpeculator(DraftModelSpeculator): self.sample_pos = torch.zeros( max_num_sampled_tokens, dtype=torch.int64, device=device ) - self.sample_idx_mapping = torch.zeros( - max_num_sampled_tokens, dtype=torch.int32, device=device + # -1 marks an inert sampling row. CUDA graph capture can execute the + # full buffer before a real batch has populated it, so zero would make + # every padding row scatter into request slot 0. + self.sample_idx_mapping = torch.full( + (max_num_sampled_tokens,), -1, dtype=torch.int32, device=device ) # [0, 1, ..., N-1, 0, 1, ..., N-1, ...] -> the per-token column index into # draft_logits[req, step, :]. @@ -255,7 +258,6 @@ class DFlashSpeculator(DraftModelSpeculator): num_tokens_across_dp, cudagraph_runtime_mode, ) - num_sample = num_reqs * self.num_speculative_steps sample_hidden_states = last_hidden_states[self.sample_indices[:num_sample]] # sample_pos is the predicted token's position Q; verification keys @@ -520,6 +522,7 @@ def _prepare_dflash_inputs_kernel( num_rejected = tl.load(num_rejected_ptr + req_idx) valid_ctx_end = ctx_end - num_rejected + num_valid_ctx = valid_ctx_end - ctx_start num_sampled = tl.load(num_sampled_ptr + req_idx) if num_sampled > 0: @@ -533,20 +536,34 @@ def _prepare_dflash_inputs_kernel( j = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) is_ctx = j < num_ctx - is_query = (j >= num_ctx) & (j < num_ctx + num_query_per_req) - query_off = j - num_ctx + is_valid_ctx = j < num_valid_ctx + is_query = (j >= num_valid_ctx) & (j < num_valid_ctx + num_query_per_req) + query_off = j - num_valid_ctx # --- Context positions / slots --- ctx_pos_idx = ctx_start + tl.where(is_ctx, j, 0) - ctx_pos = tl.load(target_positions_ptr + ctx_pos_idx, mask=is_ctx, other=0) + ctx_pos = tl.load(target_positions_ptr + ctx_pos_idx, mask=is_valid_ctx, other=0) ctx_block_num = ctx_pos // block_size ctx_block_num = tl.minimum(ctx_block_num, block_table_stride - 1) ctx_block_id = tl.load( block_table_ptr + req_idx * block_table_stride + ctx_block_num, - mask=is_ctx, + mask=is_valid_ctx, other=0, ).to(tl.int64) - ctx_slot = ctx_block_id * block_size + (ctx_pos % block_size) + # Block 0 is the null block. Old sliding-window context positions can map + # to it after eviction; rejected suffix rows are invalid context as well. + # Neither kind of row may write draft KV into physical block 0. + ctx_resident = is_valid_ctx & (ctx_block_id != 0) + ctx_slot = tl.where( + ctx_resident, + ctx_block_id * block_size + (ctx_pos % block_size), + PAD_SLOT_ID, + ) + # Stored over the full [0, num_ctx) span while the loads above are masked to + # [0, num_valid_ctx): the rejected suffix rows in between get position 0 and + # PAD_SLOT_ID. That is intentional — those rows write no KV and their + # positions are never consumed, but the span must stay fully initialized so + # a replayed graph cannot observe a stale value from an earlier batch. tl.store(out_context_positions_ptr + ctx_start + j, ctx_pos, mask=is_ctx) tl.store(out_context_slot_mapping_ptr + ctx_start + j, ctx_slot, mask=is_ctx) @@ -563,7 +580,14 @@ def _prepare_dflash_inputs_kernel( mask=is_query, other=0, ).to(tl.int64) - q_slot = q_block_id * block_size + (query_pos % block_size) + # A null block is never a writable cache slot. This can occur when a + # sliding-window block table contains evicted/global padding entries. + q_resident = is_query & (q_block_id != 0) + q_slot = tl.where( + q_resident, + q_block_id * block_size + (query_pos % block_size), + PAD_SLOT_ID, + ) tl.store(out_input_ids_ptr + query_idx, input_id, mask=is_query) clamped_query_pos = tl.minimum(query_pos, max_model_len - 1) diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index c0fb12a948..a3b00aaad2 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -91,6 +91,16 @@ from .utils import request_memory logger = init_logger(__name__) + +def _num_workspace_lanes(vllm_config: VllmConfig, use_v2_model_runner: bool) -> int: + spec_config = vllm_config.speculative_config + return ( + 2 + if use_v2_model_runner and spec_config is not None and spec_config.use_dspark() + else 1 + ) + + if TYPE_CHECKING: from vllm.device_allocator.sleep_mode_backend import SleepModeBackend from vllm.model_executor.model_loader.tensorizer import TensorizerConfig @@ -402,9 +412,13 @@ class Worker(WorkerBase): else: raise RuntimeError(f"Unsupported device type: {self.device_config.device}") - # Initialize workspace manager + # DSpark target and draft CUDA graphs retain workspace views concurrently. num_ubatches = 2 if self.vllm_config.parallel_config.enable_dbo else 1 - init_workspace_manager(self.device, num_ubatches) + init_workspace_manager( + self.device, + num_ubatches, + _num_workspace_lanes(self.vllm_config, self.use_v2_model_runner), + ) # Construct the model runner if self.use_v2_model_runner: diff --git a/vllm/v1/worker/workspace.py b/vllm/v1/worker/workspace.py index 1c502bfd8f..39b3b10349 100644 --- a/vllm/v1/worker/workspace.py +++ b/vllm/v1/worker/workspace.py @@ -3,6 +3,9 @@ import inspect import os +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar from itertools import accumulate from math import prod @@ -26,22 +29,43 @@ _GiB = 1024**3 # Global workspace manager instance _manager: "WorkspaceManager | None" = None +_workspace_lane: ContextVar[int] = ContextVar("vllm_workspace_lane", default=0) + + +@contextmanager +def use_workspace_lane(lane: int) -> Iterator[None]: + """Select an independent workspace owner for this execution context.""" + if lane < 0: + raise ValueError(f"Workspace lane must be non-negative, got {lane}.") + token = _workspace_lane.set(lane) + try: + yield + finally: + _workspace_lane.reset(token) class WorkspaceManager: """Manager for workspace allocation. - Manages one workspace buffer per active ubatch slot. + Manages one workspace buffer per active ``(ubatch, lane)`` slot. Can be locked to prevent further growth during execution. """ - def __init__(self, device: torch.device, num_ubatches: int | None = None): + def __init__( + self, + device: torch.device, + num_ubatches: int | None = None, + num_lanes: int = 1, + ): self._device = device # Cache num ubatches at init based on configuration (default to 1) self._num_ubatches = num_ubatches if num_ubatches is not None else 1 - self._current_workspaces: list[torch.Tensor | None] = [ - None - ] * self._num_ubatches + if num_lanes < 1: + raise ValueError(f"num_lanes must be at least one, got {num_lanes}.") + self._num_lanes = num_lanes + self._current_workspaces: list[torch.Tensor | None] = [None] * ( + self._num_ubatches * self._num_lanes + ) self._locked: bool = False @staticmethod @@ -126,7 +150,14 @@ class WorkspaceManager: The current workspace tensor. """ ubatch_id = dbo_current_ubatch_id() - current_workspace = self._current_workspaces[ubatch_id] + lane = _workspace_lane.get() + if lane >= self._num_lanes: + raise RuntimeError( + f"Workspace lane {lane} is not configured; manager has " + f"{self._num_lanes} lane(s)." + ) + workspace_id = ubatch_id * self._num_lanes + lane + current_workspace = self._current_workspaces[workspace_id] current_size = self._workspace_size_bytes(current_workspace) if current_size < required_bytes: @@ -161,11 +192,11 @@ class WorkspaceManager: "Workspace growth is not allowed after locking." ) - # Only resize the requesting ubatch's workspace. Other - # ubatches resize lazily on their next get_simultaneous call. + # Only resize the requesting ubatch/lane workspace. Other slots + # resize lazily on their next get_simultaneous call. # Resizing all ubatches here would orphan the other ubatch's # old tensor when it still holds views into it (DBO leak). - self._current_workspaces[ubatch_id] = None + self._current_workspaces[workspace_id] = None del current_workspace # Release the freed segment back to CUDA so the caching # allocator can reuse the GPU memory for the larger @@ -173,19 +204,20 @@ class WorkspaceManager: # dead segment in reserved memory which can cause higher peak # memory usage. torch.accelerator.empty_cache() - self._current_workspaces[ubatch_id] = torch.empty( + self._current_workspaces[workspace_id] = torch.empty( (required_bytes,), dtype=torch.uint8, device=self._device ) - current_workspace = self._current_workspaces[ubatch_id] + current_workspace = self._current_workspaces[workspace_id] if envs.VLLM_DEBUG_WORKSPACE: logger.info( "[WORKSPACE DEBUG] Resized workspace from '%s': %.2f MB -> " - "%.2f MB (ubatch %d)", + "%.2f MB (ubatch %d, lane %d)", get_caller_info(), current_size / _MB, required_bytes / _MB, ubatch_id, + lane, ) return current_workspace @@ -214,7 +246,9 @@ def current_workspace_manager() -> "WorkspaceManager": def init_workspace_manager( - device: torch.device, num_ubatches: int | None = None + device: torch.device, + num_ubatches: int | None = None, + num_lanes: int = 1, ) -> None: """Initialize the workspace manager with a device. @@ -224,6 +258,7 @@ def init_workspace_manager( Args: device: The device to allocate workspace on. num_ubatches: Number of workspace ubatch slots. Defaults to 1. + num_lanes: Number of independent execution lanes per ubatch. Defaults to 1. """ global _manager if _manager is not None: @@ -233,7 +268,7 @@ def init_workspace_manager( _manager._device, device, ) - _manager = WorkspaceManager(device, num_ubatches) + _manager = WorkspaceManager(device, num_ubatches, num_lanes) def lock_workspace() -> None: