[Bugfix] Make DSV4 sparse MLA work end-to-end for plain decode, MTP, and DSpark (#51538)

This commit is contained in:
Gabriel Wu
2026-08-15 07:01:32 -07:00
committed by GitHub
parent ac2ae8798c
commit 97388c44f9
20 changed files with 797 additions and 120 deletions
+4 -1
View File
@@ -484,7 +484,10 @@ __device__ void large_topk(const float* __restrict__ row_input,
template <uint32_t TopK, uint32_t CS>
__device__ void cooperative_topk_body(CooperativeTopKParams<TopK> params) {
const auto rank = blockIdx.y, row = blockIdx.x, tx = threadIdx.x;
const auto sl = params.lengths[row];
// Clamp at 0: `sl` is compared signed here but cast to uint32 below, so a
// negative length would otherwise emit indices 0..TopK-1 as valid instead
// of the -1 padding.
const int32_t sl = params.lengths[row] > 0 ? params.lengths[row] : 0;
int32_t* out = params.output + row * TopK;
const float* in = params.input + row * params.stride;
+20 -1
View File
@@ -906,7 +906,26 @@ __global__ void __launch_bounds__(kThreadsPerBlock, 2)
uint32_t row_idx = group_id + iter * num_groups;
if (row_idx >= params.num_rows) break;
const uint32_t seq_len = params.lengths[row_idx];
// Clamp the row length before any decision is made on it.
//
// `lengths` is int32 and is consumed here as uint32, so a negative value
// (e.g. a padded decode slot whose per-token context length underflowed)
// would reinterpret as ~4e9 and sail past every threshold below. Any
// value beyond the row width would also read into the next row.
//
// Clamping to max_seq_len additionally keeps this per-row decision
// consistent with the `cta_in_group != 0` early exit above, which is
// taken from the host-side scalar: when max_seq_len <= RADIX_THRESHOLD
// the non-leader CTAs return immediately, so a leader that reached the
// cooperative radix path would wait on the inter-CTA barrier for peers
// that no longer exist and spin until the kernel is killed.
const int32_t raw_len = params.lengths[row_idx];
const uint32_t row_bound =
params.stride < params.max_seq_len ? params.stride : params.max_seq_len;
const uint32_t non_negative_len =
raw_len > 0 ? static_cast<uint32_t>(raw_len) : 0u;
const uint32_t seq_len =
non_negative_len < row_bound ? non_negative_len : row_bound;
int32_t* row_output = params.output + row_idx * params.top_k;
const float* row_input = params.input + row_idx * params.stride;
@@ -225,6 +225,7 @@ def test_flashinfer_sparse_indices_cache(monkeypatch):
token_to_req_indices=torch.tensor([0, 1, 1], dtype=torch.int32),
decode_swa_indices=torch.tensor([[5, 6, -1, -1]], dtype=torch.int32),
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
decode_swa_width=4,
is_valid_token=torch.tensor([True], dtype=torch.bool),
num_decodes=1,
num_prefills=1,
@@ -331,3 +332,120 @@ def test_flashinfer_sparse_indices_cache(monkeypatch):
assert builder_calls == 4
assert sparse_indices_third is not sparse_indices_fourth
assert sparse_lens_third is not sparse_lens_fourth
def test_flashinfer_sparse_index_preserves_logical_window(monkeypatch):
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as flashinfer_mod
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
captured_shapes_and_windows: list[tuple[int, int]] = []
def fake_build(*args, **kwargs):
# window_size is the 12th positional arg of
# build_flashinfer_mixed_sparse_indices.
captured_shapes_and_windows.append((args[0].shape[-1], args[11]))
num_tokens = args[0].shape[0] + args[3].shape[0]
return (
torch.zeros((num_tokens, 1), dtype=torch.int32),
torch.zeros((num_tokens,), dtype=torch.int32),
)
monkeypatch.setattr(
flashinfer_mod, "build_flashinfer_mixed_sparse_indices", fake_build
)
attn = object.__new__(flashinfer_mod.DeepseekV4FlashInferMLAAttention)
attn.compress_ratio = 1
attn.window_size = 4
attn.topk_indices_buffer = torch.zeros((4, 0), dtype=torch.int32)
wide_width = 8
wide_indices = torch.full((1, wide_width), -1, dtype=torch.int32)
wide_indices[0, :2] = torch.tensor([5, 6], dtype=torch.int32)
wide_metadata = DeepseekSparseSWAMetadata(
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
slot_mapping=torch.tensor([0], dtype=torch.int64),
block_size=64,
seq_lens=torch.tensor([8], dtype=torch.int32),
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.int32),
token_to_req_indices=torch.tensor([0], dtype=torch.int32),
decode_swa_indices=wide_indices,
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
decode_swa_width=wide_width,
is_valid_token=torch.tensor([True], dtype=torch.bool),
num_decodes=1,
num_prefills=0,
num_decode_tokens=1,
num_prefill_tokens=0,
)
attn._build_sparse_index_metadata(
kv_cache=None,
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=wide_metadata,
attn_metadata=None,
swa_only=True,
)
assert captured_shapes_and_windows == [(wide_width, attn.window_size)]
empty_width = 8
empty_metadata = DeepseekSparseSWAMetadata(
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
slot_mapping=torch.tensor([0, 1], dtype=torch.int64),
block_size=64,
seq_lens=torch.tensor([8], dtype=torch.int32),
query_start_loc=torch.tensor([0, 2], dtype=torch.int32),
query_start_loc_cpu=torch.tensor([0, 2], dtype=torch.int32),
token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32),
decode_swa_indices=torch.empty((0, 1, empty_width), dtype=torch.int32),
decode_swa_lens=torch.empty((0,), dtype=torch.int32),
decode_swa_width=empty_width,
is_valid_token=torch.tensor([True, True], dtype=torch.bool),
num_decodes=0,
num_prefills=1,
num_decode_tokens=0,
num_prefill_tokens=2,
)
attn._build_sparse_index_metadata(
kv_cache=None,
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
swa_metadata=empty_metadata,
attn_metadata=None,
swa_only=True,
)
assert captured_shapes_and_windows == [
(wide_width, attn.window_size),
(empty_width, attn.window_size),
]
def test_flashinfer_mixed_sparse_indices_separates_window_and_padded_width():
from vllm.models.deepseek_v4.common.ops.cache_utils import (
build_flashinfer_mixed_sparse_indices,
)
device = torch.device("cuda")
padded_width = 8
logical_window = 4
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
decode_swa_indices=torch.empty(
(0, padded_width), dtype=torch.int32, device=device
),
decode_compressed_indices=None,
decode_compressed_topk_lens=None,
prefill_topk_indices=torch.empty((1, 0), dtype=torch.int32, device=device),
query_start_loc=torch.tensor([0, 1], dtype=torch.int32, device=device),
seq_lens=torch.tensor([logical_window], dtype=torch.int32, device=device),
token_to_req_indices=torch.tensor([0], dtype=torch.int32, device=device),
swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
swa_block_size=64,
compressed_block_table=None,
compressed_block_size=64,
window_size=logical_window,
compress_ratio=1,
topk=0,
)
assert sparse_indices.shape == (1, padded_width)
assert sparse_indices[0].cpu().tolist() == [0, 1, 2, 3, -1, -1, -1, -1]
assert sparse_lens.cpu().tolist() == [padded_width]
+95
View File
@@ -1095,6 +1095,101 @@ def test_convert_standard_mxfp4_weights_for_flashinfer_cutlass(
torch.testing.assert_close(interleaved_scales[1], w2_scale)
def test_gpt_oss_quant_config_supplies_clamped_swiglu_params():
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
from vllm.model_executor.layers.quantization.mxfp4 import GptOssMxfp4MoEMethod
fake_method = types.SimpleNamespace(
mxfp4_backend=Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
w13_precision_config=None,
w2_precision_config=None,
)
fake_layer = types.SimpleNamespace(
w13_weight_scale=torch.zeros(2, 4, 2, dtype=torch.uint8),
w2_weight_scale=torch.zeros(2, 4, 2, dtype=torch.uint8),
)
quant_config = GptOssMxfp4MoEMethod.get_fused_moe_quant_config(
fake_method, fake_layer
)
assert quant_config is not None
assert quant_config.gemm1_alpha == 1.702
assert quant_config.gemm1_beta == 1.0
assert quant_config.gemm1_clamp_limit == 7.0
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="CUDA is required for FlashInferExperts parameter storage",
)
@pytest.mark.parametrize("supplied", [False, True])
def test_flashinfer_experts_swiglu_params_follow_quant_config(supplied: bool):
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
mxfp4_w4a16_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
FlashInferExperts,
)
num_experts, intermediate_size, hidden_size = 2, 128, 256
w1_scale = torch.zeros(
(num_experts, 2 * intermediate_size, hidden_size // 32),
dtype=torch.uint8,
device="cuda",
)
w2_scale = torch.zeros(
(num_experts, hidden_size, intermediate_size // 32),
dtype=torch.uint8,
device="cuda",
)
extra = (
{"gemm1_alpha": 0.5, "gemm1_beta": 0.25, "gemm1_clamp_limit": 3.0}
if supplied
else {}
)
quant_config = mxfp4_w4a16_moe_quant_config(
w1_scale=w1_scale, w2_scale=w2_scale, **extra
)
moe_config = FusedMoEConfig(
num_experts=num_experts,
experts_per_token=1,
hidden_dim=hidden_size,
intermediate_size=intermediate_size,
num_local_experts=num_experts,
num_logical_experts=num_experts,
activation=MoEActivation.SILU,
device="cuda",
routing_method=RoutingMethodType.TopK,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=torch.bfloat16,
)
experts = FlashInferExperts(moe_config=moe_config, quant_config=quant_config)
if supplied:
for name, value in (
("gemm1_alpha", 0.5),
("gemm1_beta", 0.25),
("gemm1_clamp_limit", 3.0),
):
parameter = getattr(experts, name)
assert isinstance(parameter, torch.Tensor)
assert parameter.dtype == torch.float32
assert parameter.shape == (num_experts,)
torch.testing.assert_close(
parameter.cpu(),
torch.full((num_experts,), value, dtype=torch.float32),
)
else:
assert experts.gemm1_alpha is None
assert experts.gemm1_beta is None
assert experts.gemm1_clamp_limit is None
@pytest.mark.parametrize("topk", [1, 4])
@pytest.mark.parametrize("num_experts", [32])
@pytest.mark.parametrize("num_tokens", [1, 128])
+13
View File
@@ -31,10 +31,23 @@ from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import (
)
from vllm.models.deepseek_v4.compressor import _get_c128_boundary
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mla.compressor_utils import (
get_dspark_swa_index_width,
)
from .test_fused_indexer_q_rope_quant import quantize_to_mxfp4
@pytest.mark.parametrize(
("window_size", "num_speculative_tokens", "expected"),
[(128, 5, 192), (512, 5, 576), (1024, 0, 1024)],
)
def test_get_dspark_swa_index_width(
window_size: int, num_speculative_tokens: int, expected: int
):
assert get_dspark_swa_index_width(window_size, num_speculative_tokens) == expected
def test_compute_global_topk_reuses_output_buffers():
device = "cuda"
topk_indices = torch.tensor(
@@ -7,6 +7,9 @@ from types import SimpleNamespace
import torch
from vllm.config import set_current_vllm_config
from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import (
_required_sm120_sparse_topk,
)
from vllm.platforms.interface import DeviceCapability
from vllm.utils import flashinfer as fi_utils
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
@@ -52,3 +55,33 @@ def test_v32_glm_sm120_backend_accepts_glm_block_size(
)
assert invalid_reasons == []
def test_sm120_dsv4_capability_checks_exact_dispatch_shape(monkeypatch) -> None:
fake_module = SimpleNamespace(
_DECODE_DSV4_DISPATCH=frozenset({(32, 128), (32, 192)})
)
monkeypatch.setattr(fi_utils, "has_flashinfer_sparse_mla_sm120", lambda: True)
monkeypatch.setattr(fi_utils, "_get_submodule", lambda _name: fake_module)
fi_utils.has_flashinfer_sparse_mla_sm120_config.cache_clear()
assert fi_utils.has_flashinfer_sparse_mla_sm120_config(32, 128)
assert fi_utils.has_flashinfer_sparse_mla_sm120_config(32, 192)
assert not fi_utils.has_flashinfer_sparse_mla_sm120_config(32, 256)
assert not fi_utils.has_flashinfer_sparse_mla_sm120_config(16, 192)
fi_utils.has_flashinfer_sparse_mla_sm120_config.cache_clear()
def test_sm120_dsv4_required_topk_tracks_dspark_width() -> None:
causal = SimpleNamespace(
attention_config=SimpleNamespace(use_non_causal=False),
speculative_config=SimpleNamespace(num_speculative_tokens=5),
)
dspark = SimpleNamespace(
attention_config=SimpleNamespace(use_non_causal=True),
speculative_config=SimpleNamespace(num_speculative_tokens=5),
)
assert _required_sm120_sparse_topk(causal, 128) == 128
assert _required_sm120_sparse_topk(dspark, 128) == 192
@@ -0,0 +1,152 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import numpy as np
import pytest
import torch
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import (
prepare_dflash_inputs,
)
pytestmark = pytest.mark.skipif(
not torch.cuda.is_available(), reason="requires a CUDA device"
)
def _run_prepare(*, target_positions: list[int], block_table_values: list[int]):
device = torch.device("cuda")
max_num_reqs = 4
max_num_tokens = 16
num_speculative_steps = 3
input_buffers = SimpleNamespace(
input_ids=torch.full((max_num_tokens,), -1, dtype=torch.int32, device=device),
positions=torch.full((max_num_tokens,), -1, dtype=torch.int64, device=device),
query_start_loc=torch.full(
(max_num_reqs + 1,), -1, dtype=torch.int32, device=device
),
seq_lens=torch.full((max_num_reqs,), -1, dtype=torch.int32, device=device),
)
input_batch = SimpleNamespace(
num_reqs=1,
num_scheduled_tokens=np.array([4], dtype=np.int32),
positions=torch.tensor(target_positions, dtype=torch.int64, device=device),
query_start_loc=torch.tensor([0, 4], dtype=torch.int32, device=device),
idx_mapping=torch.tensor([2], dtype=torch.int32, device=device),
)
query_slot_mapping = torch.full(
(max_num_tokens,), -2, dtype=torch.int64, device=device
)
context_positions = torch.full(
(max_num_tokens,), -1, dtype=torch.int64, device=device
)
context_slot_mapping = torch.full(
(max_num_tokens,), -2, dtype=torch.int64, device=device
)
sample_indices = torch.full(
(max_num_reqs * num_speculative_steps,),
-1,
dtype=torch.int64,
device=device,
)
sample_pos = torch.full_like(sample_indices, -1)
sample_idx_mapping = torch.full(
sample_indices.shape, -1, dtype=torch.int32, device=device
)
temperature = torch.zeros(max_num_reqs, dtype=torch.float32, device=device)
seeds = torch.zeros(max_num_reqs, dtype=torch.int64, device=device)
input_temperature = torch.tensor(
[0.0, 0.0, 1.0, 0.0], dtype=torch.float32, device=device
)
input_seeds = torch.tensor([0, 0, 17, 0], dtype=torch.int64, device=device)
last_sampled = torch.tensor([0, 0, 99, 0], dtype=torch.int64, device=device)
next_prefill_tokens = torch.zeros_like(last_sampled)
block_table = torch.tensor([block_table_values], dtype=torch.int32, device=device)
prepare_dflash_inputs(
input_buffers,
query_slot_mapping,
context_positions,
context_slot_mapping,
sample_indices,
sample_pos,
sample_idx_mapping,
temperature,
seeds,
input_batch,
torch.tensor([1], dtype=torch.int32, device=device),
torch.tensor([2], dtype=torch.int32, device=device),
last_sampled,
next_prefill_tokens,
input_temperature,
input_seeds,
block_table,
4,
123,
num_speculative_steps,
num_speculative_steps,
max_num_reqs,
max_num_tokens,
128,
sample_from_anchor=True,
)
torch.accelerator.synchronize()
return SimpleNamespace(
input_buffers=input_buffers,
query_slot_mapping=query_slot_mapping.cpu(),
context_positions=context_positions.cpu(),
context_slot_mapping=context_slot_mapping.cpu(),
sample_indices=sample_indices.cpu(),
sample_pos=sample_pos.cpu(),
sample_idx_mapping=sample_idx_mapping.cpu(),
temperature=temperature.cpu(),
seeds=seeds.cpu(),
)
def test_prepare_dflash_inputs_excludes_rejected_context_suffix():
# Positions 10/11 use physical block 7. Rejected positions 12/13 would use
# block 8, but must be PAD context rather than contaminating draft KV.
out = _run_prepare(
target_positions=[10, 11, 12, 13],
block_table_values=[0, 0, 7, 8, 9, 10, 11, 12],
)
assert out.context_positions[:4].tolist() == [10, 11, 0, 0]
assert out.context_slot_mapping[:4].tolist() == [30, 31, PAD_SLOT_ID, PAD_SLOT_ID]
# The replacement query starts immediately after the two valid rows and
# advances from the last accepted position (11).
assert out.input_buffers.input_ids[:3].cpu().tolist() == [99, 123, 123]
assert out.input_buffers.positions[:3].cpu().tolist() == [12, 13, 14]
assert out.query_slot_mapping[:3].tolist() == [32, 33, 34]
assert out.sample_indices[:3].tolist() == [0, 1, 2]
assert out.sample_pos[:3].tolist() == [13, 14, 15]
assert out.sample_idx_mapping[:3].tolist() == [2, 2, 2]
assert out.temperature[2].item() == 1.0
assert out.seeds[2].item() == 17
def test_prepare_dflash_inputs_never_writes_the_null_block():
# The valid context uses logical block 0 and the replacement query uses
# logical block 1. Both map to the null block and must remain unwritable.
out = _run_prepare(
target_positions=[2, 3, 4, 5],
block_table_values=[0, 0, 7, 8, 9, 10, 11, 12],
)
assert out.context_slot_mapping[:4].tolist() == [
PAD_SLOT_ID,
PAD_SLOT_ID,
PAD_SLOT_ID,
PAD_SLOT_ID,
]
assert out.query_slot_mapping[:3].tolist() == [
PAD_SLOT_ID,
PAD_SLOT_ID,
PAD_SLOT_ID,
]
+102
View File
@@ -0,0 +1,102 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import cast
import pytest
import torch
import vllm.v1.worker.workspace as workspace
from vllm.config import VllmConfig
from vllm.v1.worker.gpu_worker import _num_workspace_lanes
class _SpecConfig:
def __init__(self, dspark: bool) -> None:
self._dspark = dspark
def use_dspark(self) -> bool:
return self._dspark
class _VllmConfig:
def __init__(self, spec_config: _SpecConfig | None) -> None:
self.speculative_config = spec_config
@pytest.mark.parametrize(
("use_v2_model_runner", "spec_config", "expected"),
[
(True, _SpecConfig(True), 2),
(False, _SpecConfig(True), 1),
(True, _SpecConfig(False), 1),
(True, None, 1),
],
)
def test_workspace_lane_count_is_dspark_only(
use_v2_model_runner: bool,
spec_config: _SpecConfig | None,
expected: int,
) -> None:
config = cast(VllmConfig, _VllmConfig(spec_config))
assert _num_workspace_lanes(config, use_v2_model_runner) == expected
def test_workspace_lanes_do_not_alias_and_restore_context(monkeypatch) -> None:
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: 0)
manager = workspace.WorkspaceManager(
torch.device("cpu"), num_ubatches=2, num_lanes=2
)
assert manager._current_workspaces == [None, None, None, None]
(target,) = manager.get_simultaneous(((512,), torch.uint8))
with workspace.use_workspace_lane(1):
(draft,) = manager.get_simultaneous(((256,), torch.uint8))
(draft_reused,) = manager.get_simultaneous(((8,), torch.uint8))
(target_reused,) = manager.get_simultaneous(((8,), torch.uint8))
assert manager._current_workspaces[0].numel() == 512 # type: ignore[union-attr]
assert manager._current_workspaces[1].numel() == 256 # type: ignore[union-attr]
assert manager._current_workspaces[2:] == [None, None]
assert target.data_ptr() != draft.data_ptr()
assert draft.data_ptr() == draft_reused.data_ptr()
assert target.data_ptr() == target_reused.data_ptr()
def test_workspace_lanes_compose_with_ubatches(monkeypatch) -> None:
active_ubatch = [0]
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: active_ubatch[0])
manager = workspace.WorkspaceManager(
torch.device("cpu"), num_ubatches=2, num_lanes=2
)
pointers = set()
for ubatch_id in range(2):
active_ubatch[0] = ubatch_id
for lane in range(2):
with workspace.use_workspace_lane(lane):
(buffer,) = manager.get_simultaneous(((16,), torch.uint8))
pointers.add(buffer.data_ptr())
assert len(pointers) == 4
def test_workspace_lane_validation(monkeypatch) -> None:
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: 0)
manager = workspace.WorkspaceManager(torch.device("cpu"), num_lanes=1)
with (
pytest.raises(ValueError, match="non-negative"),
workspace.use_workspace_lane(-1),
):
pass
with (
workspace.use_workspace_lane(1),
pytest.raises(RuntimeError, match="is not configured"),
):
manager.get_simultaneous(((1,), torch.uint8))
with pytest.raises(ValueError, match="at least one"):
workspace.WorkspaceManager(torch.device("cpu"), num_lanes=0)
@@ -111,21 +111,15 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
self.gemm1_alpha = _per_expert(quant_config.gemm1_alpha)
self.gemm1_beta = _per_expert(quant_config.gemm1_beta)
if quant_config.weight_quant_dtype == "mxfp4":
# This value is used specifically for gpt-oss,
# Need to revisit this for other models
if self.gemm1_alpha is None:
self.gemm1_alpha = _per_expert(1.702)
if self.gemm1_beta is None:
self.gemm1_beta = _per_expert(1.0)
if self.gemm1_clamp_limit is None:
self.gemm1_clamp_limit = _per_expert(7.0)
if quant_config.quant_dtype == "mxfp8":
self.fake_input_scale = torch.ones(
self.num_experts,
device=self.device,
dtype=torch.float32,
)
if (
quant_config.weight_quant_dtype == "mxfp4"
and quant_config.quant_dtype == "mxfp8"
):
self.fake_input_scale = torch.ones(
self.num_experts,
device=self.device,
dtype=torch.float32,
)
@property
def expects_unquantized_inputs(self) -> bool:
@@ -324,9 +318,6 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
elif self.weight_quant_dtype == "mxfp4":
assert self.w1_scale is not None and self.w2_scale is not None
assert w1.is_contiguous() and w2.is_contiguous()
assert self.gemm1_alpha is not None
assert self.gemm1_beta is not None
assert self.gemm1_clamp_limit is not None
assert topk_ids.is_contiguous()
fc1_expert_biases = self.w1_bias
+4 -4
View File
@@ -414,7 +414,9 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuild
and base.decode_swa_lens is not None
):
ragged_indices, ragged_indptr = build_ragged_indices_from_dense(
base.decode_swa_indices.reshape(base.num_decode_tokens, -1),
base.decode_swa_indices.reshape(
base.num_decode_tokens, base.decode_swa_width
),
base.decode_swa_lens,
)
ragged_indices, ragged_indptr = _copy_ragged_to_graph_buffers(
@@ -423,9 +425,7 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuild
self.decode_swa_ragged_indices_buffer,
self.decode_swa_ragged_indptr_buffer,
base.num_decode_tokens,
# Actual dense width for this build: window_size (causal) or
# noncausal_index_width (DSpark non-causal draft).
base.decode_swa_indices.shape[-1],
base.decode_swa_width,
)
return DeepseekV4ROCMAiterSparseSWAMetadata(
@@ -839,16 +839,17 @@ def build_flashinfer_mixed_sparse_indices(
) -> tuple[torch.Tensor, torch.Tensor]:
"""Build the FlashInfer DSV4 sparse-index matrix for decode-first batches.
Produces ``sparse_indices`` of shape ``[num_tokens, window_size +
padded_topk]`` (the first ``window_size`` columns are SWA slot ids, the rest
are compressed/top-k slot ids) and ``sparse_topk_lens`` (active length per
token). Decode tokens read precomputed SWA/compressed indices; prefill tokens
derive their SWA window from the position and translate local compressed
indices to global slots via the block tables.
Produces ``sparse_indices`` of shape ``[num_tokens, swa_index_width +
padded_topk]`` (the first ``swa_index_width`` columns are SWA slot ids, the
rest are compressed/top-k slot ids) and ``sparse_topk_lens`` (active length
per token). Decode tokens read precomputed SWA/compressed indices; prefill
tokens derive their SWA window from the position and translate local
compressed indices to global slots via the block tables.
"""
assert decode_swa_indices.dtype == torch.int32
assert decode_swa_indices.dim() == 2
assert decode_swa_indices.shape[-1] == window_size
swa_index_width = decode_swa_indices.shape[-1]
assert swa_index_width >= window_size
if decode_compressed_topk_lens is not None:
assert decode_compressed_topk_lens.dtype == torch.int32
assert prefill_topk_indices.dtype == torch.int32
@@ -897,7 +898,7 @@ def build_flashinfer_mixed_sparse_indices(
padded_topk = max(topk, decode_compressed_topk)
padded_topk = (padded_topk + 3) // 4 * 4
sparse_indices = torch.empty(
(num_tokens, window_size + padded_topk),
(num_tokens, swa_index_width + padded_topk),
dtype=torch.int32,
device=decode_swa_indices.device,
)
@@ -907,7 +908,7 @@ def build_flashinfer_mixed_sparse_indices(
if num_tokens == 0:
return sparse_indices, sparse_topk_lens
window_block_size = triton.next_power_of_2(max(window_size, 1))
window_block_size = triton.next_power_of_2(max(swa_index_width, 1))
topk_block_size = triton.next_power_of_2(max(padded_topk, 1))
max_block_size = max(window_block_size, topk_block_size)
num_warps = 4 if max_block_size >= 256 else 1
@@ -944,6 +945,7 @@ def build_flashinfer_mixed_sparse_indices(
compressed_span,
NUM_DECODE_TOKENS=num_decode_tokens,
WINDOW_SIZE=window_size,
SWA_INDEX_WIDTH=swa_index_width,
COMPRESS_RATIO=compress_ratio,
TOP_K=topk,
PADDED_TOP_K=padded_topk,
@@ -1011,6 +1013,7 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
compressed_block_span,
NUM_DECODE_TOKENS,
WINDOW_SIZE: tl.constexpr,
SWA_INDEX_WIDTH: tl.constexpr,
COMPRESS_RATIO: tl.constexpr,
TOP_K: tl.constexpr,
PADDED_TOP_K: tl.constexpr,
@@ -1024,9 +1027,9 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
token_idx = tl.program_id(0)
if token_idx < NUM_DECODE_TOKENS:
for i in range(0, WINDOW_SIZE, WINDOW_BLOCK_SIZE):
for i in range(0, SWA_INDEX_WIDTH, WINDOW_BLOCK_SIZE):
offset = i + tl.arange(0, WINDOW_BLOCK_SIZE)
mask = offset < WINDOW_SIZE
mask = offset < SWA_INDEX_WIDTH
values = tl.load(
decode_swa_indices_ptr + token_idx * decode_swa_stride + offset,
mask=mask,
@@ -1072,7 +1075,7 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
tl.store(
sparse_indices_ptr
+ token_idx * sparse_indices_stride
+ WINDOW_SIZE
+ SWA_INDEX_WIDTH
+ offset,
values,
mask=mask,
@@ -1086,7 +1089,7 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
else:
compressed_len = tl.full((), DECODE_COMPRESSED_TOPK, dtype=tl.int32)
tl.store(sparse_topk_lens_ptr + token_idx, WINDOW_SIZE + compressed_len)
tl.store(sparse_topk_lens_ptr + token_idx, SWA_INDEX_WIDTH + compressed_len)
return
prefill_idx = token_idx - NUM_DECODE_TOKENS
@@ -1102,9 +1105,9 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
swa_start_pos = pos - swa_len + 1
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K)
for i in range(0, WINDOW_SIZE, WINDOW_BLOCK_SIZE):
for i in range(0, SWA_INDEX_WIDTH, WINDOW_BLOCK_SIZE):
offset = i + tl.arange(0, WINDOW_BLOCK_SIZE)
mask = offset < WINDOW_SIZE
mask = offset < SWA_INDEX_WIDTH
pos_offset = swa_start_pos + offset
block_indices = pos_offset // swa_block_size
block_numbers = tl.load(
@@ -1148,10 +1151,10 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
tl.store(
sparse_indices_ptr
+ token_idx * sparse_indices_stride
+ WINDOW_SIZE
+ SWA_INDEX_WIDTH
+ offset,
slot_ids,
mask=mask,
)
tl.store(sparse_topk_lens_ptr + token_idx, WINDOW_SIZE + topk_len)
tl.store(sparse_topk_lens_ptr + token_idx, SWA_INDEX_WIDTH + topk_len)
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, ClassVar, cast
import torch
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
@@ -25,6 +26,9 @@ from vllm.platforms import current_platform
from vllm.platforms.interface import DeviceCapability
from vllm.utils.flashinfer import flashinfer_trtllm_batch_decode_sparse_mla_dsv4
from vllm.v1.attention.backend import MultipleOf
from vllm.v1.attention.backends.mla.compressor_utils import (
get_dspark_swa_index_width,
)
if TYPE_CHECKING:
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
@@ -74,6 +78,19 @@ def _pad_to_supported_q_heads(num_heads: int) -> int:
)
def _required_sm120_sparse_topk(vllm_config: VllmConfig, window_size: int) -> int:
"""Return the SM120 DSV4 SWA specialization needed by this model."""
if not vllm_config.attention_config.use_non_causal:
return window_size
speculative_config = vllm_config.speculative_config
if speculative_config is None:
return window_size
return get_dspark_swa_index_width(
window_size,
speculative_config.num_speculative_tokens,
)
class DeepseekV4FlashInferMLASparseBackend(DeepseekV4SparseMLABackend):
"""FlashInfer backend using the DSv4 sparse metadata/cache layout.
@@ -309,7 +326,7 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
assert swa_metadata.block_table is not None
decode_swa_indices = swa_metadata.decode_swa_indices.reshape(
num_decode_tokens, self.window_size
num_decode_tokens, swa_metadata.decode_swa_width
)
decode_compressed_topk_lens = None
decode_compressed_indices_are_local = False
@@ -568,14 +585,18 @@ class DeepseekV4FlashInferSM120Attention(DeepseekV4Attention):
tma_aligned_scales=self._tma_aligned_scales,
)
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
from vllm.utils.flashinfer import has_flashinfer_sparse_mla_sm120
def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None:
super().__init__(vllm_config, *args, **kwargs)
from vllm.utils.flashinfer import has_flashinfer_sparse_mla_sm120_config
if not has_flashinfer_sparse_mla_sm120():
required_topk = _required_sm120_sparse_topk(vllm_config, self.window_size)
if not has_flashinfer_sparse_mla_sm120_config(self.padded_heads, required_topk):
raise RuntimeError(
"FLASHINFER_MLA_SPARSE_DSV4 on SM120 requires FlashInfer's "
"sparse MLA decode API."
"FLASHINFER_MLA_SPARSE_DSV4 on SM120 requires a FlashInfer "
"DSV4 sparse MLA decode specialization for "
f"(num_q_heads={self.padded_heads}, top_k={required_topk}). "
"Install a FlashInfer build containing "
"flashinfer-ai/flashinfer#4380."
)
self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe()
# Per-tensor FP8 cache path scales.
+16
View File
@@ -237,6 +237,22 @@ def has_flashinfer_sparse_mla_sm120() -> bool:
)
@functools.cache
def has_flashinfer_sparse_mla_sm120_config(num_q_heads: int, top_k: int) -> bool:
"""Return whether FlashInfer ships an SM120 DSV4 decode specialization.
The public sparse MLA API predates some DSV4 shapes, so checking only that
the callable exists can select a package that later aborts or rejects a
valid vLLM configuration. Inspect FlashInfer's dispatch table until it
exposes a public capability query.
"""
if not has_flashinfer_sparse_mla_sm120():
return False
mod = _get_submodule("flashinfer.mla._sparse_mla_sm120")
dispatch = getattr(mod, "_DECODE_DSV4_DISPATCH", None) if mod else None
return dispatch is not None and (int(num_q_heads), int(top_k)) in dispatch
@functools.cache
def has_flashinfer_cutedsl() -> bool:
"""Return ``True`` if FlashInfer cutedsl module is available."""
@@ -3,6 +3,18 @@
import torch
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import cdiv
_DSPARK_SWA_INDEX_ALIGNMENT = 64
def get_dspark_swa_index_width(
window_size: int,
num_speculative_tokens: int,
) -> int:
"""Return the padded width of non-causal DSpark SWA indices."""
width = max(int(window_size), 0) + max(int(num_speculative_tokens), 0)
return cdiv(width, _DSPARK_SWA_INDEX_ALIGNMENT) * _DSPARK_SWA_INDEX_ALIGNMENT
@triton.jit
+9 -3
View File
@@ -57,9 +57,12 @@ def _prepare_uniform_decode_kernel(
req_id = idx // max_decode_len
local_idx = idx % max_decode_len
# Compute number of KVs attended to by this token.
# Compute number of KVs attended to by this token. Padding requests have
# seq_len == 0, which would otherwise make the first token of each padded
# request negative (e.g. next_n=2 gives 0-2+0+1 = -1). Downstream kernels
# read these as uint32, turning -1 into ~4e9.
seq_len = tl.load(seq_lens_ptr + req_id)
per_token_seq_len = seq_len - max_decode_len + local_idx + 1
per_token_seq_len = tl.maximum(seq_len - max_decode_len + local_idx + 1, 0)
tl.store(decode_seq_lens_ptr + idx, per_token_seq_len)
# Copy block table row.
@@ -748,12 +751,15 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
seq_lens_buffer = self.decode_seq_lens_buffer[
: num_decodes * max_decode_len
].view(num_decodes, max_decode_len)
# Clamp at 0: padding requests have seq_len == 0, which would
# otherwise make token 0 negative (next_n=2 gives 0-2+1+0 = -1).
# Downstream kernels read these as uint32, turning -1 into ~4e9.
seq_lens_buffer[:] = (
seq_lens.unsqueeze(1)
- max_decode_len
+ 1
+ self.offsets_buffer[:max_decode_len]
)
).clamp_(min=0)
seq_lens = seq_lens_buffer
return seq_lens, block_table, decode_lens, num_decodes, requires_padding
+16 -4
View File
@@ -22,6 +22,9 @@ from vllm.v1.attention.backend import (
CommonAttentionMetadata,
MultipleOf,
)
from vllm.v1.attention.backends.mla.compressor_utils import (
get_dspark_swa_index_width,
)
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.attention.ops.flashmla import FlashMLASchedMeta, get_mla_metadata
from vllm.v1.kv_cache_interface import (
@@ -174,8 +177,10 @@ class DeepseekSparseSWAMetadata:
is_valid_token: torch.Tensor | None = None # [num_tokens]
token_to_req_indices: torch.Tensor | None = None # [num_tokens]
decode_swa_indices: torch.Tensor | None = None # [num_decode_tokens, window_size]
decode_swa_indices: torch.Tensor | None = None # [num_decode_tokens, width]
decode_swa_lens: torch.Tensor | None = None # [num_decode_tokens]
# window_size (causal) or noncausal_index_width (DSpark non-causal).
decode_swa_width: int = 0
# Paged-coordinate prefill SWA indices/lens (FP8 paged-direct prefill).
prefill_swa_indices: torch.Tensor | None = (
None # [num_prefill_tokens, 1, window_size]
@@ -487,11 +492,14 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
# DSpark draft: the block is non-causal (every query attends to the
# trailing window of context PLUS all query tokens, including future ones),
# so its per-token index list is wider than `window_size`. The kernel pads
# the q-head count to B_TOPK (64/128), which requires the index width to be
# a multiple of 128.
# the q-head count to B_TOPK. Pad to a kernel-supported width; the logical
# SWA window remains unchanged when the padded matrix is built.
self.is_dspark = spec_config is not None and spec_config.use_dspark()
self.noncausal_index_width = (
cdiv(self.window_size + self.num_speculative_tokens, 128) * 128
get_dspark_swa_index_width(
self.window_size,
self.num_speculative_tokens,
)
if self.is_dspark
else 0
)
@@ -536,6 +544,9 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
is_valid_token.copy_(slot_mapping >= 0)
non_causal = not common_attn_metadata.causal
decode_swa_width = (
self.noncausal_index_width if non_causal else self.window_size
)
decode_swa_indices = self.decode_swa_indices
if num_decode_tokens > 0:
self.decode_swa_lens[num_decode_tokens:] = 0
@@ -634,6 +645,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
token_to_req_indices=token_to_req_indices,
decode_swa_indices=decode_swa_indices[:num_decode_tokens],
decode_swa_lens=self.decode_swa_lens[:num_decode_tokens],
decode_swa_width=decode_swa_width,
prefill_swa_indices=(
self.prefill_swa_indices[:num_prefill_tokens]
if num_prefill_tokens > 0
+48 -40
View File
@@ -149,6 +149,7 @@ from vllm.v1.worker.utils import (
copy_kv_cache_blocks_inplace,
get_uniform_decode_token_count,
)
from vllm.v1.worker.workspace import use_workspace_lane
logger = init_logger(__name__)
@@ -164,6 +165,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.parallel_config = vllm_config.parallel_config
self.scheduler_config = vllm_config.scheduler_config
self.speculative_config = vllm_config.speculative_config
self._draft_workspace_lane = int(
self.speculative_config is not None and self.speculative_config.use_dspark()
)
self.observability_config = vllm_config.observability_config
self.device = device
@@ -364,10 +368,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
assert self.speculative_config is not None
set_eagle3_aux_hidden_state_layers(self.model, self.speculative_config)
if isinstance(self.speculator, DraftModelSpeculator):
self.speculator.load_model(self.model)
eplb_models_added = self.eplb.maybe_register_speculator(
self.speculator, self.speculative_config, load_dummy_weights
)
with use_workspace_lane(self._draft_workspace_lane):
self.speculator.load_model(self.model)
eplb_models_added = self.eplb.maybe_register_speculator(
self.speculator, self.speculative_config, load_dummy_weights
)
time_after_load = time.perf_counter()
self.model_memory_usage = m.consumed_memory
@@ -732,27 +737,28 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if hasattr(self.model, "get_mtp_target_hidden_states"):
pre_hc_hidden_states = self.model.get_mtp_target_hidden_states()
spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr]
self.speculator.propose(
input_batch=input_batch,
attn_metadata=attn_metadata,
slot_mappings=slot_mappings_by_layer,
last_hidden_states=spec_hidden_states,
aux_hidden_states=aux_hidden_states,
num_sampled=torch.ones(
input_batch.num_reqs, dtype=torch.int32, device=self.device
),
num_rejected=torch.zeros(
input_batch.num_reqs, dtype=torch.int32, device=self.device
),
last_sampled=self.req_states.last_sampled_tokens,
next_prefill_tokens=self.req_states.next_prefill_tokens,
temperature=self.sampler.sampling_states.temperature.gpu,
seeds=self.sampler.sampling_states.seeds.gpu,
dummy_run=True,
skip_attn_for_dummy_run=skip_attn,
mm_inputs=mm_inputs,
is_profile=is_profile,
)
with use_workspace_lane(self._draft_workspace_lane):
self.speculator.propose(
input_batch=input_batch,
attn_metadata=attn_metadata,
slot_mappings=slot_mappings_by_layer,
last_hidden_states=spec_hidden_states,
aux_hidden_states=aux_hidden_states,
num_sampled=torch.ones(
input_batch.num_reqs, dtype=torch.int32, device=self.device
),
num_rejected=torch.zeros(
input_batch.num_reqs, dtype=torch.int32, device=self.device
),
last_sampled=self.req_states.last_sampled_tokens,
next_prefill_tokens=self.req_states.next_prefill_tokens,
temperature=self.sampler.sampling_states.temperature.gpu,
seeds=self.sampler.sampling_states.seeds.gpu,
dummy_run=True,
skip_attn_for_dummy_run=skip_attn,
mm_inputs=mm_inputs,
is_profile=is_profile,
)
self.step_timing.drafter_end()
assert hidden_states is not None # Last PP rank always has hidden_states
@@ -880,7 +886,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
lora_capture_hook=create_lora_capture_hook(self.lora_config, self),
)
if self.speculator is not None:
self.speculator.capture()
with use_workspace_lane(self._draft_workspace_lane):
self.speculator.capture()
if self.adaptive_verification is not None:
with self.step_timing.collect() as timings:
for batch in self.adaptive_verification.batches_to_profile(
@@ -1799,20 +1806,21 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if hasattr(self.model, "get_mtp_target_hidden_states"):
pre_hc_hidden_states = self.model.get_mtp_target_hidden_states()
spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr]
draft_tokens = self.speculator.propose(
input_batch,
attn_metadata,
slot_mappings_by_layer,
spec_hidden_states,
aux_hidden_states,
num_sampled,
num_rejected,
self.req_states.last_sampled_tokens,
self.req_states.next_prefill_tokens,
self.sampler.sampling_states.temperature.gpu,
self.sampler.sampling_states.seeds.gpu,
mm_inputs=mm_inputs,
)
with use_workspace_lane(self._draft_workspace_lane):
draft_tokens = self.speculator.propose(
input_batch,
attn_metadata,
slot_mappings_by_layer,
spec_hidden_states,
aux_hidden_states,
num_sampled,
num_rejected,
self.req_states.last_sampled_tokens,
self.req_states.next_prefill_tokens,
self.sampler.sampling_states.temperature.gpu,
self.sampler.sampling_states.seeds.gpu,
mm_inputs=mm_inputs,
)
self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens
if self.adaptive_verification is not None:
self.adaptive_verification.record_confidences(
@@ -84,8 +84,11 @@ class DFlashSpeculator(DraftModelSpeculator):
self.sample_pos = torch.zeros(
max_num_sampled_tokens, dtype=torch.int64, device=device
)
self.sample_idx_mapping = torch.zeros(
max_num_sampled_tokens, dtype=torch.int32, device=device
# -1 marks an inert sampling row. CUDA graph capture can execute the
# full buffer before a real batch has populated it, so zero would make
# every padding row scatter into request slot 0.
self.sample_idx_mapping = torch.full(
(max_num_sampled_tokens,), -1, dtype=torch.int32, device=device
)
# [0, 1, ..., N-1, 0, 1, ..., N-1, ...] -> the per-token column index into
# draft_logits[req, step, :].
@@ -255,7 +258,6 @@ class DFlashSpeculator(DraftModelSpeculator):
num_tokens_across_dp,
cudagraph_runtime_mode,
)
num_sample = num_reqs * self.num_speculative_steps
sample_hidden_states = last_hidden_states[self.sample_indices[:num_sample]]
# sample_pos is the predicted token's position Q; verification keys
@@ -520,6 +522,7 @@ def _prepare_dflash_inputs_kernel(
num_rejected = tl.load(num_rejected_ptr + req_idx)
valid_ctx_end = ctx_end - num_rejected
num_valid_ctx = valid_ctx_end - ctx_start
num_sampled = tl.load(num_sampled_ptr + req_idx)
if num_sampled > 0:
@@ -533,20 +536,34 @@ def _prepare_dflash_inputs_kernel(
j = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
is_ctx = j < num_ctx
is_query = (j >= num_ctx) & (j < num_ctx + num_query_per_req)
query_off = j - num_ctx
is_valid_ctx = j < num_valid_ctx
is_query = (j >= num_valid_ctx) & (j < num_valid_ctx + num_query_per_req)
query_off = j - num_valid_ctx
# --- Context positions / slots ---
ctx_pos_idx = ctx_start + tl.where(is_ctx, j, 0)
ctx_pos = tl.load(target_positions_ptr + ctx_pos_idx, mask=is_ctx, other=0)
ctx_pos = tl.load(target_positions_ptr + ctx_pos_idx, mask=is_valid_ctx, other=0)
ctx_block_num = ctx_pos // block_size
ctx_block_num = tl.minimum(ctx_block_num, block_table_stride - 1)
ctx_block_id = tl.load(
block_table_ptr + req_idx * block_table_stride + ctx_block_num,
mask=is_ctx,
mask=is_valid_ctx,
other=0,
).to(tl.int64)
ctx_slot = ctx_block_id * block_size + (ctx_pos % block_size)
# Block 0 is the null block. Old sliding-window context positions can map
# to it after eviction; rejected suffix rows are invalid context as well.
# Neither kind of row may write draft KV into physical block 0.
ctx_resident = is_valid_ctx & (ctx_block_id != 0)
ctx_slot = tl.where(
ctx_resident,
ctx_block_id * block_size + (ctx_pos % block_size),
PAD_SLOT_ID,
)
# Stored over the full [0, num_ctx) span while the loads above are masked to
# [0, num_valid_ctx): the rejected suffix rows in between get position 0 and
# PAD_SLOT_ID. That is intentional — those rows write no KV and their
# positions are never consumed, but the span must stay fully initialized so
# a replayed graph cannot observe a stale value from an earlier batch.
tl.store(out_context_positions_ptr + ctx_start + j, ctx_pos, mask=is_ctx)
tl.store(out_context_slot_mapping_ptr + ctx_start + j, ctx_slot, mask=is_ctx)
@@ -563,7 +580,14 @@ def _prepare_dflash_inputs_kernel(
mask=is_query,
other=0,
).to(tl.int64)
q_slot = q_block_id * block_size + (query_pos % block_size)
# A null block is never a writable cache slot. This can occur when a
# sliding-window block table contains evicted/global padding entries.
q_resident = is_query & (q_block_id != 0)
q_slot = tl.where(
q_resident,
q_block_id * block_size + (query_pos % block_size),
PAD_SLOT_ID,
)
tl.store(out_input_ids_ptr + query_idx, input_id, mask=is_query)
clamped_query_pos = tl.minimum(query_pos, max_model_len - 1)
+16 -2
View File
@@ -91,6 +91,16 @@ from .utils import request_memory
logger = init_logger(__name__)
def _num_workspace_lanes(vllm_config: VllmConfig, use_v2_model_runner: bool) -> int:
spec_config = vllm_config.speculative_config
return (
2
if use_v2_model_runner and spec_config is not None and spec_config.use_dspark()
else 1
)
if TYPE_CHECKING:
from vllm.device_allocator.sleep_mode_backend import SleepModeBackend
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
@@ -402,9 +412,13 @@ class Worker(WorkerBase):
else:
raise RuntimeError(f"Unsupported device type: {self.device_config.device}")
# Initialize workspace manager
# DSpark target and draft CUDA graphs retain workspace views concurrently.
num_ubatches = 2 if self.vllm_config.parallel_config.enable_dbo else 1
init_workspace_manager(self.device, num_ubatches)
init_workspace_manager(
self.device,
num_ubatches,
_num_workspace_lanes(self.vllm_config, self.use_v2_model_runner),
)
# Construct the model runner
if self.use_v2_model_runner:
+49 -14
View File
@@ -3,6 +3,9 @@
import inspect
import os
from collections.abc import Iterator
from contextlib import contextmanager
from contextvars import ContextVar
from itertools import accumulate
from math import prod
@@ -26,22 +29,43 @@ _GiB = 1024**3
# Global workspace manager instance
_manager: "WorkspaceManager | None" = None
_workspace_lane: ContextVar[int] = ContextVar("vllm_workspace_lane", default=0)
@contextmanager
def use_workspace_lane(lane: int) -> Iterator[None]:
"""Select an independent workspace owner for this execution context."""
if lane < 0:
raise ValueError(f"Workspace lane must be non-negative, got {lane}.")
token = _workspace_lane.set(lane)
try:
yield
finally:
_workspace_lane.reset(token)
class WorkspaceManager:
"""Manager for workspace allocation.
Manages one workspace buffer per active ubatch slot.
Manages one workspace buffer per active ``(ubatch, lane)`` slot.
Can be locked to prevent further growth during execution.
"""
def __init__(self, device: torch.device, num_ubatches: int | None = None):
def __init__(
self,
device: torch.device,
num_ubatches: int | None = None,
num_lanes: int = 1,
):
self._device = device
# Cache num ubatches at init based on configuration (default to 1)
self._num_ubatches = num_ubatches if num_ubatches is not None else 1
self._current_workspaces: list[torch.Tensor | None] = [
None
] * self._num_ubatches
if num_lanes < 1:
raise ValueError(f"num_lanes must be at least one, got {num_lanes}.")
self._num_lanes = num_lanes
self._current_workspaces: list[torch.Tensor | None] = [None] * (
self._num_ubatches * self._num_lanes
)
self._locked: bool = False
@staticmethod
@@ -126,7 +150,14 @@ class WorkspaceManager:
The current workspace tensor.
"""
ubatch_id = dbo_current_ubatch_id()
current_workspace = self._current_workspaces[ubatch_id]
lane = _workspace_lane.get()
if lane >= self._num_lanes:
raise RuntimeError(
f"Workspace lane {lane} is not configured; manager has "
f"{self._num_lanes} lane(s)."
)
workspace_id = ubatch_id * self._num_lanes + lane
current_workspace = self._current_workspaces[workspace_id]
current_size = self._workspace_size_bytes(current_workspace)
if current_size < required_bytes:
@@ -161,11 +192,11 @@ class WorkspaceManager:
"Workspace growth is not allowed after locking."
)
# Only resize the requesting ubatch's workspace. Other
# ubatches resize lazily on their next get_simultaneous call.
# Only resize the requesting ubatch/lane workspace. Other slots
# resize lazily on their next get_simultaneous call.
# Resizing all ubatches here would orphan the other ubatch's
# old tensor when it still holds views into it (DBO leak).
self._current_workspaces[ubatch_id] = None
self._current_workspaces[workspace_id] = None
del current_workspace
# Release the freed segment back to CUDA so the caching
# allocator can reuse the GPU memory for the larger
@@ -173,19 +204,20 @@ class WorkspaceManager:
# dead segment in reserved memory which can cause higher peak
# memory usage.
torch.accelerator.empty_cache()
self._current_workspaces[ubatch_id] = torch.empty(
self._current_workspaces[workspace_id] = torch.empty(
(required_bytes,), dtype=torch.uint8, device=self._device
)
current_workspace = self._current_workspaces[ubatch_id]
current_workspace = self._current_workspaces[workspace_id]
if envs.VLLM_DEBUG_WORKSPACE:
logger.info(
"[WORKSPACE DEBUG] Resized workspace from '%s': %.2f MB -> "
"%.2f MB (ubatch %d)",
"%.2f MB (ubatch %d, lane %d)",
get_caller_info(),
current_size / _MB,
required_bytes / _MB,
ubatch_id,
lane,
)
return current_workspace
@@ -214,7 +246,9 @@ def current_workspace_manager() -> "WorkspaceManager":
def init_workspace_manager(
device: torch.device, num_ubatches: int | None = None
device: torch.device,
num_ubatches: int | None = None,
num_lanes: int = 1,
) -> None:
"""Initialize the workspace manager with a device.
@@ -224,6 +258,7 @@ def init_workspace_manager(
Args:
device: The device to allocate workspace on.
num_ubatches: Number of workspace ubatch slots. Defaults to 1.
num_lanes: Number of independent execution lanes per ubatch. Defaults to 1.
"""
global _manager
if _manager is not None:
@@ -233,7 +268,7 @@ def init_workspace_manager(
_manager._device,
device,
)
_manager = WorkspaceManager(device, num_ubatches)
_manager = WorkspaceManager(device, num_ubatches, num_lanes)
def lock_workspace() -> None: