[Bugfix] Make DSV4 sparse MLA work end-to-end for plain decode, MTP, and DSpark (#51538)
This commit is contained in:
@@ -484,7 +484,10 @@ __device__ void large_topk(const float* __restrict__ row_input,
|
||||
template <uint32_t TopK, uint32_t CS>
|
||||
__device__ void cooperative_topk_body(CooperativeTopKParams<TopK> params) {
|
||||
const auto rank = blockIdx.y, row = blockIdx.x, tx = threadIdx.x;
|
||||
const auto sl = params.lengths[row];
|
||||
// Clamp at 0: `sl` is compared signed here but cast to uint32 below, so a
|
||||
// negative length would otherwise emit indices 0..TopK-1 as valid instead
|
||||
// of the -1 padding.
|
||||
const int32_t sl = params.lengths[row] > 0 ? params.lengths[row] : 0;
|
||||
int32_t* out = params.output + row * TopK;
|
||||
const float* in = params.input + row * params.stride;
|
||||
|
||||
|
||||
@@ -906,7 +906,26 @@ __global__ void __launch_bounds__(kThreadsPerBlock, 2)
|
||||
uint32_t row_idx = group_id + iter * num_groups;
|
||||
if (row_idx >= params.num_rows) break;
|
||||
|
||||
const uint32_t seq_len = params.lengths[row_idx];
|
||||
// Clamp the row length before any decision is made on it.
|
||||
//
|
||||
// `lengths` is int32 and is consumed here as uint32, so a negative value
|
||||
// (e.g. a padded decode slot whose per-token context length underflowed)
|
||||
// would reinterpret as ~4e9 and sail past every threshold below. Any
|
||||
// value beyond the row width would also read into the next row.
|
||||
//
|
||||
// Clamping to max_seq_len additionally keeps this per-row decision
|
||||
// consistent with the `cta_in_group != 0` early exit above, which is
|
||||
// taken from the host-side scalar: when max_seq_len <= RADIX_THRESHOLD
|
||||
// the non-leader CTAs return immediately, so a leader that reached the
|
||||
// cooperative radix path would wait on the inter-CTA barrier for peers
|
||||
// that no longer exist and spin until the kernel is killed.
|
||||
const int32_t raw_len = params.lengths[row_idx];
|
||||
const uint32_t row_bound =
|
||||
params.stride < params.max_seq_len ? params.stride : params.max_seq_len;
|
||||
const uint32_t non_negative_len =
|
||||
raw_len > 0 ? static_cast<uint32_t>(raw_len) : 0u;
|
||||
const uint32_t seq_len =
|
||||
non_negative_len < row_bound ? non_negative_len : row_bound;
|
||||
int32_t* row_output = params.output + row_idx * params.top_k;
|
||||
const float* row_input = params.input + row_idx * params.stride;
|
||||
|
||||
|
||||
@@ -225,6 +225,7 @@ def test_flashinfer_sparse_indices_cache(monkeypatch):
|
||||
token_to_req_indices=torch.tensor([0, 1, 1], dtype=torch.int32),
|
||||
decode_swa_indices=torch.tensor([[5, 6, -1, -1]], dtype=torch.int32),
|
||||
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
|
||||
decode_swa_width=4,
|
||||
is_valid_token=torch.tensor([True], dtype=torch.bool),
|
||||
num_decodes=1,
|
||||
num_prefills=1,
|
||||
@@ -331,3 +332,120 @@ def test_flashinfer_sparse_indices_cache(monkeypatch):
|
||||
assert builder_calls == 4
|
||||
assert sparse_indices_third is not sparse_indices_fourth
|
||||
assert sparse_lens_third is not sparse_lens_fourth
|
||||
|
||||
|
||||
def test_flashinfer_sparse_index_preserves_logical_window(monkeypatch):
|
||||
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as flashinfer_mod
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
captured_shapes_and_windows: list[tuple[int, int]] = []
|
||||
|
||||
def fake_build(*args, **kwargs):
|
||||
# window_size is the 12th positional arg of
|
||||
# build_flashinfer_mixed_sparse_indices.
|
||||
captured_shapes_and_windows.append((args[0].shape[-1], args[11]))
|
||||
num_tokens = args[0].shape[0] + args[3].shape[0]
|
||||
return (
|
||||
torch.zeros((num_tokens, 1), dtype=torch.int32),
|
||||
torch.zeros((num_tokens,), dtype=torch.int32),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
flashinfer_mod, "build_flashinfer_mixed_sparse_indices", fake_build
|
||||
)
|
||||
|
||||
attn = object.__new__(flashinfer_mod.DeepseekV4FlashInferMLAAttention)
|
||||
attn.compress_ratio = 1
|
||||
attn.window_size = 4
|
||||
attn.topk_indices_buffer = torch.zeros((4, 0), dtype=torch.int32)
|
||||
|
||||
wide_width = 8
|
||||
wide_indices = torch.full((1, wide_width), -1, dtype=torch.int32)
|
||||
wide_indices[0, :2] = torch.tensor([5, 6], dtype=torch.int32)
|
||||
wide_metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
|
||||
slot_mapping=torch.tensor([0], dtype=torch.int64),
|
||||
block_size=64,
|
||||
seq_lens=torch.tensor([8], dtype=torch.int32),
|
||||
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
|
||||
query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.int32),
|
||||
token_to_req_indices=torch.tensor([0], dtype=torch.int32),
|
||||
decode_swa_indices=wide_indices,
|
||||
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
|
||||
decode_swa_width=wide_width,
|
||||
is_valid_token=torch.tensor([True], dtype=torch.bool),
|
||||
num_decodes=1,
|
||||
num_prefills=0,
|
||||
num_decode_tokens=1,
|
||||
num_prefill_tokens=0,
|
||||
)
|
||||
attn._build_sparse_index_metadata(
|
||||
kv_cache=None,
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=wide_metadata,
|
||||
attn_metadata=None,
|
||||
swa_only=True,
|
||||
)
|
||||
assert captured_shapes_and_windows == [(wide_width, attn.window_size)]
|
||||
|
||||
empty_width = 8
|
||||
empty_metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
|
||||
slot_mapping=torch.tensor([0, 1], dtype=torch.int64),
|
||||
block_size=64,
|
||||
seq_lens=torch.tensor([8], dtype=torch.int32),
|
||||
query_start_loc=torch.tensor([0, 2], dtype=torch.int32),
|
||||
query_start_loc_cpu=torch.tensor([0, 2], dtype=torch.int32),
|
||||
token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32),
|
||||
decode_swa_indices=torch.empty((0, 1, empty_width), dtype=torch.int32),
|
||||
decode_swa_lens=torch.empty((0,), dtype=torch.int32),
|
||||
decode_swa_width=empty_width,
|
||||
is_valid_token=torch.tensor([True, True], dtype=torch.bool),
|
||||
num_decodes=0,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=0,
|
||||
num_prefill_tokens=2,
|
||||
)
|
||||
attn._build_sparse_index_metadata(
|
||||
kv_cache=None,
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=empty_metadata,
|
||||
attn_metadata=None,
|
||||
swa_only=True,
|
||||
)
|
||||
assert captured_shapes_and_windows == [
|
||||
(wide_width, attn.window_size),
|
||||
(empty_width, attn.window_size),
|
||||
]
|
||||
|
||||
|
||||
def test_flashinfer_mixed_sparse_indices_separates_window_and_padded_width():
|
||||
from vllm.models.deepseek_v4.common.ops.cache_utils import (
|
||||
build_flashinfer_mixed_sparse_indices,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
padded_width = 8
|
||||
logical_window = 4
|
||||
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
|
||||
decode_swa_indices=torch.empty(
|
||||
(0, padded_width), dtype=torch.int32, device=device
|
||||
),
|
||||
decode_compressed_indices=None,
|
||||
decode_compressed_topk_lens=None,
|
||||
prefill_topk_indices=torch.empty((1, 0), dtype=torch.int32, device=device),
|
||||
query_start_loc=torch.tensor([0, 1], dtype=torch.int32, device=device),
|
||||
seq_lens=torch.tensor([logical_window], dtype=torch.int32, device=device),
|
||||
token_to_req_indices=torch.tensor([0], dtype=torch.int32, device=device),
|
||||
swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
|
||||
swa_block_size=64,
|
||||
compressed_block_table=None,
|
||||
compressed_block_size=64,
|
||||
window_size=logical_window,
|
||||
compress_ratio=1,
|
||||
topk=0,
|
||||
)
|
||||
|
||||
assert sparse_indices.shape == (1, padded_width)
|
||||
assert sparse_indices[0].cpu().tolist() == [0, 1, 2, 3, -1, -1, -1, -1]
|
||||
assert sparse_lens.cpu().tolist() == [padded_width]
|
||||
|
||||
@@ -1095,6 +1095,101 @@ def test_convert_standard_mxfp4_weights_for_flashinfer_cutlass(
|
||||
torch.testing.assert_close(interleaved_scales[1], w2_scale)
|
||||
|
||||
|
||||
def test_gpt_oss_quant_config_supplies_clamped_swiglu_params():
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
|
||||
from vllm.model_executor.layers.quantization.mxfp4 import GptOssMxfp4MoEMethod
|
||||
|
||||
fake_method = types.SimpleNamespace(
|
||||
mxfp4_backend=Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
||||
w13_precision_config=None,
|
||||
w2_precision_config=None,
|
||||
)
|
||||
fake_layer = types.SimpleNamespace(
|
||||
w13_weight_scale=torch.zeros(2, 4, 2, dtype=torch.uint8),
|
||||
w2_weight_scale=torch.zeros(2, 4, 2, dtype=torch.uint8),
|
||||
)
|
||||
|
||||
quant_config = GptOssMxfp4MoEMethod.get_fused_moe_quant_config(
|
||||
fake_method, fake_layer
|
||||
)
|
||||
|
||||
assert quant_config is not None
|
||||
assert quant_config.gemm1_alpha == 1.702
|
||||
assert quant_config.gemm1_beta == 1.0
|
||||
assert quant_config.gemm1_clamp_limit == 7.0
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="CUDA is required for FlashInferExperts parameter storage",
|
||||
)
|
||||
@pytest.mark.parametrize("supplied", [False, True])
|
||||
def test_flashinfer_experts_swiglu_params_follow_quant_config(supplied: bool):
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
mxfp4_w4a16_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
|
||||
FlashInferExperts,
|
||||
)
|
||||
|
||||
num_experts, intermediate_size, hidden_size = 2, 128, 256
|
||||
w1_scale = torch.zeros(
|
||||
(num_experts, 2 * intermediate_size, hidden_size // 32),
|
||||
dtype=torch.uint8,
|
||||
device="cuda",
|
||||
)
|
||||
w2_scale = torch.zeros(
|
||||
(num_experts, hidden_size, intermediate_size // 32),
|
||||
dtype=torch.uint8,
|
||||
device="cuda",
|
||||
)
|
||||
extra = (
|
||||
{"gemm1_alpha": 0.5, "gemm1_beta": 0.25, "gemm1_clamp_limit": 3.0}
|
||||
if supplied
|
||||
else {}
|
||||
)
|
||||
quant_config = mxfp4_w4a16_moe_quant_config(
|
||||
w1_scale=w1_scale, w2_scale=w2_scale, **extra
|
||||
)
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=num_experts,
|
||||
experts_per_token=1,
|
||||
hidden_dim=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
num_local_experts=num_experts,
|
||||
num_logical_experts=num_experts,
|
||||
activation=MoEActivation.SILU,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=torch.bfloat16,
|
||||
)
|
||||
experts = FlashInferExperts(moe_config=moe_config, quant_config=quant_config)
|
||||
|
||||
if supplied:
|
||||
for name, value in (
|
||||
("gemm1_alpha", 0.5),
|
||||
("gemm1_beta", 0.25),
|
||||
("gemm1_clamp_limit", 3.0),
|
||||
):
|
||||
parameter = getattr(experts, name)
|
||||
assert isinstance(parameter, torch.Tensor)
|
||||
assert parameter.dtype == torch.float32
|
||||
assert parameter.shape == (num_experts,)
|
||||
torch.testing.assert_close(
|
||||
parameter.cpu(),
|
||||
torch.full((num_experts,), value, dtype=torch.float32),
|
||||
)
|
||||
else:
|
||||
assert experts.gemm1_alpha is None
|
||||
assert experts.gemm1_beta is None
|
||||
assert experts.gemm1_clamp_limit is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("topk", [1, 4])
|
||||
@pytest.mark.parametrize("num_experts", [32])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 128])
|
||||
|
||||
@@ -31,10 +31,23 @@ from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import (
|
||||
)
|
||||
from vllm.models.deepseek_v4.compressor import _get_c128_boundary
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
get_dspark_swa_index_width,
|
||||
)
|
||||
|
||||
from .test_fused_indexer_q_rope_quant import quantize_to_mxfp4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("window_size", "num_speculative_tokens", "expected"),
|
||||
[(128, 5, 192), (512, 5, 576), (1024, 0, 1024)],
|
||||
)
|
||||
def test_get_dspark_swa_index_width(
|
||||
window_size: int, num_speculative_tokens: int, expected: int
|
||||
):
|
||||
assert get_dspark_swa_index_width(window_size, num_speculative_tokens) == expected
|
||||
|
||||
|
||||
def test_compute_global_topk_reuses_output_buffers():
|
||||
device = "cuda"
|
||||
topk_indices = torch.tensor(
|
||||
|
||||
@@ -7,6 +7,9 @@ from types import SimpleNamespace
|
||||
import torch
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import (
|
||||
_required_sm120_sparse_topk,
|
||||
)
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.utils import flashinfer as fi_utils
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
@@ -52,3 +55,33 @@ def test_v32_glm_sm120_backend_accepts_glm_block_size(
|
||||
)
|
||||
|
||||
assert invalid_reasons == []
|
||||
|
||||
|
||||
def test_sm120_dsv4_capability_checks_exact_dispatch_shape(monkeypatch) -> None:
|
||||
fake_module = SimpleNamespace(
|
||||
_DECODE_DSV4_DISPATCH=frozenset({(32, 128), (32, 192)})
|
||||
)
|
||||
monkeypatch.setattr(fi_utils, "has_flashinfer_sparse_mla_sm120", lambda: True)
|
||||
monkeypatch.setattr(fi_utils, "_get_submodule", lambda _name: fake_module)
|
||||
fi_utils.has_flashinfer_sparse_mla_sm120_config.cache_clear()
|
||||
|
||||
assert fi_utils.has_flashinfer_sparse_mla_sm120_config(32, 128)
|
||||
assert fi_utils.has_flashinfer_sparse_mla_sm120_config(32, 192)
|
||||
assert not fi_utils.has_flashinfer_sparse_mla_sm120_config(32, 256)
|
||||
assert not fi_utils.has_flashinfer_sparse_mla_sm120_config(16, 192)
|
||||
|
||||
fi_utils.has_flashinfer_sparse_mla_sm120_config.cache_clear()
|
||||
|
||||
|
||||
def test_sm120_dsv4_required_topk_tracks_dspark_width() -> None:
|
||||
causal = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(use_non_causal=False),
|
||||
speculative_config=SimpleNamespace(num_speculative_tokens=5),
|
||||
)
|
||||
dspark = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(use_non_causal=True),
|
||||
speculative_config=SimpleNamespace(num_speculative_tokens=5),
|
||||
)
|
||||
|
||||
assert _required_sm120_sparse_topk(causal, 128) == 128
|
||||
assert _required_sm120_sparse_topk(dspark, 128) == 192
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import (
|
||||
prepare_dflash_inputs,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="requires a CUDA device"
|
||||
)
|
||||
|
||||
|
||||
def _run_prepare(*, target_positions: list[int], block_table_values: list[int]):
|
||||
device = torch.device("cuda")
|
||||
max_num_reqs = 4
|
||||
max_num_tokens = 16
|
||||
num_speculative_steps = 3
|
||||
|
||||
input_buffers = SimpleNamespace(
|
||||
input_ids=torch.full((max_num_tokens,), -1, dtype=torch.int32, device=device),
|
||||
positions=torch.full((max_num_tokens,), -1, dtype=torch.int64, device=device),
|
||||
query_start_loc=torch.full(
|
||||
(max_num_reqs + 1,), -1, dtype=torch.int32, device=device
|
||||
),
|
||||
seq_lens=torch.full((max_num_reqs,), -1, dtype=torch.int32, device=device),
|
||||
)
|
||||
input_batch = SimpleNamespace(
|
||||
num_reqs=1,
|
||||
num_scheduled_tokens=np.array([4], dtype=np.int32),
|
||||
positions=torch.tensor(target_positions, dtype=torch.int64, device=device),
|
||||
query_start_loc=torch.tensor([0, 4], dtype=torch.int32, device=device),
|
||||
idx_mapping=torch.tensor([2], dtype=torch.int32, device=device),
|
||||
)
|
||||
query_slot_mapping = torch.full(
|
||||
(max_num_tokens,), -2, dtype=torch.int64, device=device
|
||||
)
|
||||
context_positions = torch.full(
|
||||
(max_num_tokens,), -1, dtype=torch.int64, device=device
|
||||
)
|
||||
context_slot_mapping = torch.full(
|
||||
(max_num_tokens,), -2, dtype=torch.int64, device=device
|
||||
)
|
||||
sample_indices = torch.full(
|
||||
(max_num_reqs * num_speculative_steps,),
|
||||
-1,
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
sample_pos = torch.full_like(sample_indices, -1)
|
||||
sample_idx_mapping = torch.full(
|
||||
sample_indices.shape, -1, dtype=torch.int32, device=device
|
||||
)
|
||||
temperature = torch.zeros(max_num_reqs, dtype=torch.float32, device=device)
|
||||
seeds = torch.zeros(max_num_reqs, dtype=torch.int64, device=device)
|
||||
input_temperature = torch.tensor(
|
||||
[0.0, 0.0, 1.0, 0.0], dtype=torch.float32, device=device
|
||||
)
|
||||
input_seeds = torch.tensor([0, 0, 17, 0], dtype=torch.int64, device=device)
|
||||
last_sampled = torch.tensor([0, 0, 99, 0], dtype=torch.int64, device=device)
|
||||
next_prefill_tokens = torch.zeros_like(last_sampled)
|
||||
block_table = torch.tensor([block_table_values], dtype=torch.int32, device=device)
|
||||
|
||||
prepare_dflash_inputs(
|
||||
input_buffers,
|
||||
query_slot_mapping,
|
||||
context_positions,
|
||||
context_slot_mapping,
|
||||
sample_indices,
|
||||
sample_pos,
|
||||
sample_idx_mapping,
|
||||
temperature,
|
||||
seeds,
|
||||
input_batch,
|
||||
torch.tensor([1], dtype=torch.int32, device=device),
|
||||
torch.tensor([2], dtype=torch.int32, device=device),
|
||||
last_sampled,
|
||||
next_prefill_tokens,
|
||||
input_temperature,
|
||||
input_seeds,
|
||||
block_table,
|
||||
4,
|
||||
123,
|
||||
num_speculative_steps,
|
||||
num_speculative_steps,
|
||||
max_num_reqs,
|
||||
max_num_tokens,
|
||||
128,
|
||||
sample_from_anchor=True,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
return SimpleNamespace(
|
||||
input_buffers=input_buffers,
|
||||
query_slot_mapping=query_slot_mapping.cpu(),
|
||||
context_positions=context_positions.cpu(),
|
||||
context_slot_mapping=context_slot_mapping.cpu(),
|
||||
sample_indices=sample_indices.cpu(),
|
||||
sample_pos=sample_pos.cpu(),
|
||||
sample_idx_mapping=sample_idx_mapping.cpu(),
|
||||
temperature=temperature.cpu(),
|
||||
seeds=seeds.cpu(),
|
||||
)
|
||||
|
||||
|
||||
def test_prepare_dflash_inputs_excludes_rejected_context_suffix():
|
||||
# Positions 10/11 use physical block 7. Rejected positions 12/13 would use
|
||||
# block 8, but must be PAD context rather than contaminating draft KV.
|
||||
out = _run_prepare(
|
||||
target_positions=[10, 11, 12, 13],
|
||||
block_table_values=[0, 0, 7, 8, 9, 10, 11, 12],
|
||||
)
|
||||
|
||||
assert out.context_positions[:4].tolist() == [10, 11, 0, 0]
|
||||
assert out.context_slot_mapping[:4].tolist() == [30, 31, PAD_SLOT_ID, PAD_SLOT_ID]
|
||||
|
||||
# The replacement query starts immediately after the two valid rows and
|
||||
# advances from the last accepted position (11).
|
||||
assert out.input_buffers.input_ids[:3].cpu().tolist() == [99, 123, 123]
|
||||
assert out.input_buffers.positions[:3].cpu().tolist() == [12, 13, 14]
|
||||
assert out.query_slot_mapping[:3].tolist() == [32, 33, 34]
|
||||
assert out.sample_indices[:3].tolist() == [0, 1, 2]
|
||||
assert out.sample_pos[:3].tolist() == [13, 14, 15]
|
||||
assert out.sample_idx_mapping[:3].tolist() == [2, 2, 2]
|
||||
assert out.temperature[2].item() == 1.0
|
||||
assert out.seeds[2].item() == 17
|
||||
|
||||
|
||||
def test_prepare_dflash_inputs_never_writes_the_null_block():
|
||||
# The valid context uses logical block 0 and the replacement query uses
|
||||
# logical block 1. Both map to the null block and must remain unwritable.
|
||||
out = _run_prepare(
|
||||
target_positions=[2, 3, 4, 5],
|
||||
block_table_values=[0, 0, 7, 8, 9, 10, 11, 12],
|
||||
)
|
||||
|
||||
assert out.context_slot_mapping[:4].tolist() == [
|
||||
PAD_SLOT_ID,
|
||||
PAD_SLOT_ID,
|
||||
PAD_SLOT_ID,
|
||||
PAD_SLOT_ID,
|
||||
]
|
||||
assert out.query_slot_mapping[:3].tolist() == [
|
||||
PAD_SLOT_ID,
|
||||
PAD_SLOT_ID,
|
||||
PAD_SLOT_ID,
|
||||
]
|
||||
@@ -0,0 +1,102 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.worker.workspace as workspace
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.worker.gpu_worker import _num_workspace_lanes
|
||||
|
||||
|
||||
class _SpecConfig:
|
||||
def __init__(self, dspark: bool) -> None:
|
||||
self._dspark = dspark
|
||||
|
||||
def use_dspark(self) -> bool:
|
||||
return self._dspark
|
||||
|
||||
|
||||
class _VllmConfig:
|
||||
def __init__(self, spec_config: _SpecConfig | None) -> None:
|
||||
self.speculative_config = spec_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("use_v2_model_runner", "spec_config", "expected"),
|
||||
[
|
||||
(True, _SpecConfig(True), 2),
|
||||
(False, _SpecConfig(True), 1),
|
||||
(True, _SpecConfig(False), 1),
|
||||
(True, None, 1),
|
||||
],
|
||||
)
|
||||
def test_workspace_lane_count_is_dspark_only(
|
||||
use_v2_model_runner: bool,
|
||||
spec_config: _SpecConfig | None,
|
||||
expected: int,
|
||||
) -> None:
|
||||
config = cast(VllmConfig, _VllmConfig(spec_config))
|
||||
assert _num_workspace_lanes(config, use_v2_model_runner) == expected
|
||||
|
||||
|
||||
def test_workspace_lanes_do_not_alias_and_restore_context(monkeypatch) -> None:
|
||||
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: 0)
|
||||
manager = workspace.WorkspaceManager(
|
||||
torch.device("cpu"), num_ubatches=2, num_lanes=2
|
||||
)
|
||||
|
||||
assert manager._current_workspaces == [None, None, None, None]
|
||||
|
||||
(target,) = manager.get_simultaneous(((512,), torch.uint8))
|
||||
with workspace.use_workspace_lane(1):
|
||||
(draft,) = manager.get_simultaneous(((256,), torch.uint8))
|
||||
(draft_reused,) = manager.get_simultaneous(((8,), torch.uint8))
|
||||
(target_reused,) = manager.get_simultaneous(((8,), torch.uint8))
|
||||
|
||||
assert manager._current_workspaces[0].numel() == 512 # type: ignore[union-attr]
|
||||
assert manager._current_workspaces[1].numel() == 256 # type: ignore[union-attr]
|
||||
assert manager._current_workspaces[2:] == [None, None]
|
||||
assert target.data_ptr() != draft.data_ptr()
|
||||
assert draft.data_ptr() == draft_reused.data_ptr()
|
||||
assert target.data_ptr() == target_reused.data_ptr()
|
||||
|
||||
|
||||
def test_workspace_lanes_compose_with_ubatches(monkeypatch) -> None:
|
||||
active_ubatch = [0]
|
||||
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: active_ubatch[0])
|
||||
manager = workspace.WorkspaceManager(
|
||||
torch.device("cpu"), num_ubatches=2, num_lanes=2
|
||||
)
|
||||
|
||||
pointers = set()
|
||||
for ubatch_id in range(2):
|
||||
active_ubatch[0] = ubatch_id
|
||||
for lane in range(2):
|
||||
with workspace.use_workspace_lane(lane):
|
||||
(buffer,) = manager.get_simultaneous(((16,), torch.uint8))
|
||||
pointers.add(buffer.data_ptr())
|
||||
|
||||
assert len(pointers) == 4
|
||||
|
||||
|
||||
def test_workspace_lane_validation(monkeypatch) -> None:
|
||||
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: 0)
|
||||
manager = workspace.WorkspaceManager(torch.device("cpu"), num_lanes=1)
|
||||
|
||||
with (
|
||||
pytest.raises(ValueError, match="non-negative"),
|
||||
workspace.use_workspace_lane(-1),
|
||||
):
|
||||
pass
|
||||
|
||||
with (
|
||||
workspace.use_workspace_lane(1),
|
||||
pytest.raises(RuntimeError, match="is not configured"),
|
||||
):
|
||||
manager.get_simultaneous(((1,), torch.uint8))
|
||||
|
||||
with pytest.raises(ValueError, match="at least one"):
|
||||
workspace.WorkspaceManager(torch.device("cpu"), num_lanes=0)
|
||||
@@ -111,21 +111,15 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
|
||||
self.gemm1_alpha = _per_expert(quant_config.gemm1_alpha)
|
||||
self.gemm1_beta = _per_expert(quant_config.gemm1_beta)
|
||||
|
||||
if quant_config.weight_quant_dtype == "mxfp4":
|
||||
# This value is used specifically for gpt-oss,
|
||||
# Need to revisit this for other models
|
||||
if self.gemm1_alpha is None:
|
||||
self.gemm1_alpha = _per_expert(1.702)
|
||||
if self.gemm1_beta is None:
|
||||
self.gemm1_beta = _per_expert(1.0)
|
||||
if self.gemm1_clamp_limit is None:
|
||||
self.gemm1_clamp_limit = _per_expert(7.0)
|
||||
if quant_config.quant_dtype == "mxfp8":
|
||||
self.fake_input_scale = torch.ones(
|
||||
self.num_experts,
|
||||
device=self.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
if (
|
||||
quant_config.weight_quant_dtype == "mxfp4"
|
||||
and quant_config.quant_dtype == "mxfp8"
|
||||
):
|
||||
self.fake_input_scale = torch.ones(
|
||||
self.num_experts,
|
||||
device=self.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
@@ -324,9 +318,6 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
|
||||
elif self.weight_quant_dtype == "mxfp4":
|
||||
assert self.w1_scale is not None and self.w2_scale is not None
|
||||
assert w1.is_contiguous() and w2.is_contiguous()
|
||||
assert self.gemm1_alpha is not None
|
||||
assert self.gemm1_beta is not None
|
||||
assert self.gemm1_clamp_limit is not None
|
||||
assert topk_ids.is_contiguous()
|
||||
|
||||
fc1_expert_biases = self.w1_bias
|
||||
|
||||
@@ -414,7 +414,9 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuild
|
||||
and base.decode_swa_lens is not None
|
||||
):
|
||||
ragged_indices, ragged_indptr = build_ragged_indices_from_dense(
|
||||
base.decode_swa_indices.reshape(base.num_decode_tokens, -1),
|
||||
base.decode_swa_indices.reshape(
|
||||
base.num_decode_tokens, base.decode_swa_width
|
||||
),
|
||||
base.decode_swa_lens,
|
||||
)
|
||||
ragged_indices, ragged_indptr = _copy_ragged_to_graph_buffers(
|
||||
@@ -423,9 +425,7 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekSparseSWAMetadataBuild
|
||||
self.decode_swa_ragged_indices_buffer,
|
||||
self.decode_swa_ragged_indptr_buffer,
|
||||
base.num_decode_tokens,
|
||||
# Actual dense width for this build: window_size (causal) or
|
||||
# noncausal_index_width (DSpark non-causal draft).
|
||||
base.decode_swa_indices.shape[-1],
|
||||
base.decode_swa_width,
|
||||
)
|
||||
|
||||
return DeepseekV4ROCMAiterSparseSWAMetadata(
|
||||
|
||||
@@ -839,16 +839,17 @@ def build_flashinfer_mixed_sparse_indices(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Build the FlashInfer DSV4 sparse-index matrix for decode-first batches.
|
||||
|
||||
Produces ``sparse_indices`` of shape ``[num_tokens, window_size +
|
||||
padded_topk]`` (the first ``window_size`` columns are SWA slot ids, the rest
|
||||
are compressed/top-k slot ids) and ``sparse_topk_lens`` (active length per
|
||||
token). Decode tokens read precomputed SWA/compressed indices; prefill tokens
|
||||
derive their SWA window from the position and translate local compressed
|
||||
indices to global slots via the block tables.
|
||||
Produces ``sparse_indices`` of shape ``[num_tokens, swa_index_width +
|
||||
padded_topk]`` (the first ``swa_index_width`` columns are SWA slot ids, the
|
||||
rest are compressed/top-k slot ids) and ``sparse_topk_lens`` (active length
|
||||
per token). Decode tokens read precomputed SWA/compressed indices; prefill
|
||||
tokens derive their SWA window from the position and translate local
|
||||
compressed indices to global slots via the block tables.
|
||||
"""
|
||||
assert decode_swa_indices.dtype == torch.int32
|
||||
assert decode_swa_indices.dim() == 2
|
||||
assert decode_swa_indices.shape[-1] == window_size
|
||||
swa_index_width = decode_swa_indices.shape[-1]
|
||||
assert swa_index_width >= window_size
|
||||
if decode_compressed_topk_lens is not None:
|
||||
assert decode_compressed_topk_lens.dtype == torch.int32
|
||||
assert prefill_topk_indices.dtype == torch.int32
|
||||
@@ -897,7 +898,7 @@ def build_flashinfer_mixed_sparse_indices(
|
||||
padded_topk = max(topk, decode_compressed_topk)
|
||||
padded_topk = (padded_topk + 3) // 4 * 4
|
||||
sparse_indices = torch.empty(
|
||||
(num_tokens, window_size + padded_topk),
|
||||
(num_tokens, swa_index_width + padded_topk),
|
||||
dtype=torch.int32,
|
||||
device=decode_swa_indices.device,
|
||||
)
|
||||
@@ -907,7 +908,7 @@ def build_flashinfer_mixed_sparse_indices(
|
||||
if num_tokens == 0:
|
||||
return sparse_indices, sparse_topk_lens
|
||||
|
||||
window_block_size = triton.next_power_of_2(max(window_size, 1))
|
||||
window_block_size = triton.next_power_of_2(max(swa_index_width, 1))
|
||||
topk_block_size = triton.next_power_of_2(max(padded_topk, 1))
|
||||
max_block_size = max(window_block_size, topk_block_size)
|
||||
num_warps = 4 if max_block_size >= 256 else 1
|
||||
@@ -944,6 +945,7 @@ def build_flashinfer_mixed_sparse_indices(
|
||||
compressed_span,
|
||||
NUM_DECODE_TOKENS=num_decode_tokens,
|
||||
WINDOW_SIZE=window_size,
|
||||
SWA_INDEX_WIDTH=swa_index_width,
|
||||
COMPRESS_RATIO=compress_ratio,
|
||||
TOP_K=topk,
|
||||
PADDED_TOP_K=padded_topk,
|
||||
@@ -1011,6 +1013,7 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
|
||||
compressed_block_span,
|
||||
NUM_DECODE_TOKENS,
|
||||
WINDOW_SIZE: tl.constexpr,
|
||||
SWA_INDEX_WIDTH: tl.constexpr,
|
||||
COMPRESS_RATIO: tl.constexpr,
|
||||
TOP_K: tl.constexpr,
|
||||
PADDED_TOP_K: tl.constexpr,
|
||||
@@ -1024,9 +1027,9 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
|
||||
token_idx = tl.program_id(0)
|
||||
|
||||
if token_idx < NUM_DECODE_TOKENS:
|
||||
for i in range(0, WINDOW_SIZE, WINDOW_BLOCK_SIZE):
|
||||
for i in range(0, SWA_INDEX_WIDTH, WINDOW_BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, WINDOW_BLOCK_SIZE)
|
||||
mask = offset < WINDOW_SIZE
|
||||
mask = offset < SWA_INDEX_WIDTH
|
||||
values = tl.load(
|
||||
decode_swa_indices_ptr + token_idx * decode_swa_stride + offset,
|
||||
mask=mask,
|
||||
@@ -1072,7 +1075,7 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
|
||||
tl.store(
|
||||
sparse_indices_ptr
|
||||
+ token_idx * sparse_indices_stride
|
||||
+ WINDOW_SIZE
|
||||
+ SWA_INDEX_WIDTH
|
||||
+ offset,
|
||||
values,
|
||||
mask=mask,
|
||||
@@ -1086,7 +1089,7 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
|
||||
else:
|
||||
compressed_len = tl.full((), DECODE_COMPRESSED_TOPK, dtype=tl.int32)
|
||||
|
||||
tl.store(sparse_topk_lens_ptr + token_idx, WINDOW_SIZE + compressed_len)
|
||||
tl.store(sparse_topk_lens_ptr + token_idx, SWA_INDEX_WIDTH + compressed_len)
|
||||
return
|
||||
|
||||
prefill_idx = token_idx - NUM_DECODE_TOKENS
|
||||
@@ -1102,9 +1105,9 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
|
||||
swa_start_pos = pos - swa_len + 1
|
||||
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K)
|
||||
|
||||
for i in range(0, WINDOW_SIZE, WINDOW_BLOCK_SIZE):
|
||||
for i in range(0, SWA_INDEX_WIDTH, WINDOW_BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, WINDOW_BLOCK_SIZE)
|
||||
mask = offset < WINDOW_SIZE
|
||||
mask = offset < SWA_INDEX_WIDTH
|
||||
pos_offset = swa_start_pos + offset
|
||||
block_indices = pos_offset // swa_block_size
|
||||
block_numbers = tl.load(
|
||||
@@ -1148,10 +1151,10 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
|
||||
tl.store(
|
||||
sparse_indices_ptr
|
||||
+ token_idx * sparse_indices_stride
|
||||
+ WINDOW_SIZE
|
||||
+ SWA_INDEX_WIDTH
|
||||
+ offset,
|
||||
slot_ids,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
tl.store(sparse_topk_lens_ptr + token_idx, WINDOW_SIZE + topk_len)
|
||||
tl.store(sparse_topk_lens_ptr + token_idx, SWA_INDEX_WIDTH + topk_len)
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, ClassVar, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.models.deepseek_v4.attention import DeepseekV4Attention
|
||||
@@ -25,6 +26,9 @@ from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.utils.flashinfer import flashinfer_trtllm_batch_decode_sparse_mla_dsv4
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
get_dspark_swa_index_width,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
@@ -74,6 +78,19 @@ def _pad_to_supported_q_heads(num_heads: int) -> int:
|
||||
)
|
||||
|
||||
|
||||
def _required_sm120_sparse_topk(vllm_config: VllmConfig, window_size: int) -> int:
|
||||
"""Return the SM120 DSV4 SWA specialization needed by this model."""
|
||||
if not vllm_config.attention_config.use_non_causal:
|
||||
return window_size
|
||||
speculative_config = vllm_config.speculative_config
|
||||
if speculative_config is None:
|
||||
return window_size
|
||||
return get_dspark_swa_index_width(
|
||||
window_size,
|
||||
speculative_config.num_speculative_tokens,
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4FlashInferMLASparseBackend(DeepseekV4SparseMLABackend):
|
||||
"""FlashInfer backend using the DSv4 sparse metadata/cache layout.
|
||||
|
||||
@@ -309,7 +326,7 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
assert swa_metadata.block_table is not None
|
||||
|
||||
decode_swa_indices = swa_metadata.decode_swa_indices.reshape(
|
||||
num_decode_tokens, self.window_size
|
||||
num_decode_tokens, swa_metadata.decode_swa_width
|
||||
)
|
||||
decode_compressed_topk_lens = None
|
||||
decode_compressed_indices_are_local = False
|
||||
@@ -568,14 +585,18 @@ class DeepseekV4FlashInferSM120Attention(DeepseekV4Attention):
|
||||
tma_aligned_scales=self._tma_aligned_scales,
|
||||
)
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
from vllm.utils.flashinfer import has_flashinfer_sparse_mla_sm120
|
||||
def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None:
|
||||
super().__init__(vllm_config, *args, **kwargs)
|
||||
from vllm.utils.flashinfer import has_flashinfer_sparse_mla_sm120_config
|
||||
|
||||
if not has_flashinfer_sparse_mla_sm120():
|
||||
required_topk = _required_sm120_sparse_topk(vllm_config, self.window_size)
|
||||
if not has_flashinfer_sparse_mla_sm120_config(self.padded_heads, required_topk):
|
||||
raise RuntimeError(
|
||||
"FLASHINFER_MLA_SPARSE_DSV4 on SM120 requires FlashInfer's "
|
||||
"sparse MLA decode API."
|
||||
"FLASHINFER_MLA_SPARSE_DSV4 on SM120 requires a FlashInfer "
|
||||
"DSV4 sparse MLA decode specialization for "
|
||||
f"(num_q_heads={self.padded_heads}, top_k={required_topk}). "
|
||||
"Install a FlashInfer build containing "
|
||||
"flashinfer-ai/flashinfer#4380."
|
||||
)
|
||||
self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe()
|
||||
# Per-tensor FP8 cache path scales.
|
||||
|
||||
@@ -237,6 +237,22 @@ def has_flashinfer_sparse_mla_sm120() -> bool:
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def has_flashinfer_sparse_mla_sm120_config(num_q_heads: int, top_k: int) -> bool:
|
||||
"""Return whether FlashInfer ships an SM120 DSV4 decode specialization.
|
||||
|
||||
The public sparse MLA API predates some DSV4 shapes, so checking only that
|
||||
the callable exists can select a package that later aborts or rejects a
|
||||
valid vLLM configuration. Inspect FlashInfer's dispatch table until it
|
||||
exposes a public capability query.
|
||||
"""
|
||||
if not has_flashinfer_sparse_mla_sm120():
|
||||
return False
|
||||
mod = _get_submodule("flashinfer.mla._sparse_mla_sm120")
|
||||
dispatch = getattr(mod, "_DECODE_DSV4_DISPATCH", None) if mod else None
|
||||
return dispatch is not None and (int(num_q_heads), int(top_k)) in dispatch
|
||||
|
||||
|
||||
@functools.cache
|
||||
def has_flashinfer_cutedsl() -> bool:
|
||||
"""Return ``True`` if FlashInfer cutedsl module is available."""
|
||||
|
||||
@@ -3,6 +3,18 @@
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
_DSPARK_SWA_INDEX_ALIGNMENT = 64
|
||||
|
||||
|
||||
def get_dspark_swa_index_width(
|
||||
window_size: int,
|
||||
num_speculative_tokens: int,
|
||||
) -> int:
|
||||
"""Return the padded width of non-causal DSpark SWA indices."""
|
||||
width = max(int(window_size), 0) + max(int(num_speculative_tokens), 0)
|
||||
return cdiv(width, _DSPARK_SWA_INDEX_ALIGNMENT) * _DSPARK_SWA_INDEX_ALIGNMENT
|
||||
|
||||
|
||||
@triton.jit
|
||||
|
||||
@@ -57,9 +57,12 @@ def _prepare_uniform_decode_kernel(
|
||||
req_id = idx // max_decode_len
|
||||
local_idx = idx % max_decode_len
|
||||
|
||||
# Compute number of KVs attended to by this token.
|
||||
# Compute number of KVs attended to by this token. Padding requests have
|
||||
# seq_len == 0, which would otherwise make the first token of each padded
|
||||
# request negative (e.g. next_n=2 gives 0-2+0+1 = -1). Downstream kernels
|
||||
# read these as uint32, turning -1 into ~4e9.
|
||||
seq_len = tl.load(seq_lens_ptr + req_id)
|
||||
per_token_seq_len = seq_len - max_decode_len + local_idx + 1
|
||||
per_token_seq_len = tl.maximum(seq_len - max_decode_len + local_idx + 1, 0)
|
||||
tl.store(decode_seq_lens_ptr + idx, per_token_seq_len)
|
||||
|
||||
# Copy block table row.
|
||||
@@ -748,12 +751,15 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder):
|
||||
seq_lens_buffer = self.decode_seq_lens_buffer[
|
||||
: num_decodes * max_decode_len
|
||||
].view(num_decodes, max_decode_len)
|
||||
# Clamp at 0: padding requests have seq_len == 0, which would
|
||||
# otherwise make token 0 negative (next_n=2 gives 0-2+1+0 = -1).
|
||||
# Downstream kernels read these as uint32, turning -1 into ~4e9.
|
||||
seq_lens_buffer[:] = (
|
||||
seq_lens.unsqueeze(1)
|
||||
- max_decode_len
|
||||
+ 1
|
||||
+ self.offsets_buffer[:max_decode_len]
|
||||
)
|
||||
).clamp_(min=0)
|
||||
seq_lens = seq_lens_buffer
|
||||
return seq_lens, block_table, decode_lens, num_decodes, requires_padding
|
||||
|
||||
|
||||
@@ -22,6 +22,9 @@ from vllm.v1.attention.backend import (
|
||||
CommonAttentionMetadata,
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
get_dspark_swa_index_width,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.attention.ops.flashmla import FlashMLASchedMeta, get_mla_metadata
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -174,8 +177,10 @@ class DeepseekSparseSWAMetadata:
|
||||
|
||||
is_valid_token: torch.Tensor | None = None # [num_tokens]
|
||||
token_to_req_indices: torch.Tensor | None = None # [num_tokens]
|
||||
decode_swa_indices: torch.Tensor | None = None # [num_decode_tokens, window_size]
|
||||
decode_swa_indices: torch.Tensor | None = None # [num_decode_tokens, width]
|
||||
decode_swa_lens: torch.Tensor | None = None # [num_decode_tokens]
|
||||
# window_size (causal) or noncausal_index_width (DSpark non-causal).
|
||||
decode_swa_width: int = 0
|
||||
# Paged-coordinate prefill SWA indices/lens (FP8 paged-direct prefill).
|
||||
prefill_swa_indices: torch.Tensor | None = (
|
||||
None # [num_prefill_tokens, 1, window_size]
|
||||
@@ -487,11 +492,14 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
||||
# DSpark draft: the block is non-causal (every query attends to the
|
||||
# trailing window of context PLUS all query tokens, including future ones),
|
||||
# so its per-token index list is wider than `window_size`. The kernel pads
|
||||
# the q-head count to B_TOPK (64/128), which requires the index width to be
|
||||
# a multiple of 128.
|
||||
# the q-head count to B_TOPK. Pad to a kernel-supported width; the logical
|
||||
# SWA window remains unchanged when the padded matrix is built.
|
||||
self.is_dspark = spec_config is not None and spec_config.use_dspark()
|
||||
self.noncausal_index_width = (
|
||||
cdiv(self.window_size + self.num_speculative_tokens, 128) * 128
|
||||
get_dspark_swa_index_width(
|
||||
self.window_size,
|
||||
self.num_speculative_tokens,
|
||||
)
|
||||
if self.is_dspark
|
||||
else 0
|
||||
)
|
||||
@@ -536,6 +544,9 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
||||
is_valid_token.copy_(slot_mapping >= 0)
|
||||
|
||||
non_causal = not common_attn_metadata.causal
|
||||
decode_swa_width = (
|
||||
self.noncausal_index_width if non_causal else self.window_size
|
||||
)
|
||||
decode_swa_indices = self.decode_swa_indices
|
||||
if num_decode_tokens > 0:
|
||||
self.decode_swa_lens[num_decode_tokens:] = 0
|
||||
@@ -634,6 +645,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder):
|
||||
token_to_req_indices=token_to_req_indices,
|
||||
decode_swa_indices=decode_swa_indices[:num_decode_tokens],
|
||||
decode_swa_lens=self.decode_swa_lens[:num_decode_tokens],
|
||||
decode_swa_width=decode_swa_width,
|
||||
prefill_swa_indices=(
|
||||
self.prefill_swa_indices[:num_prefill_tokens]
|
||||
if num_prefill_tokens > 0
|
||||
|
||||
@@ -149,6 +149,7 @@ from vllm.v1.worker.utils import (
|
||||
copy_kv_cache_blocks_inplace,
|
||||
get_uniform_decode_token_count,
|
||||
)
|
||||
from vllm.v1.worker.workspace import use_workspace_lane
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -164,6 +165,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.parallel_config = vllm_config.parallel_config
|
||||
self.scheduler_config = vllm_config.scheduler_config
|
||||
self.speculative_config = vllm_config.speculative_config
|
||||
self._draft_workspace_lane = int(
|
||||
self.speculative_config is not None and self.speculative_config.use_dspark()
|
||||
)
|
||||
self.observability_config = vllm_config.observability_config
|
||||
|
||||
self.device = device
|
||||
@@ -364,10 +368,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
assert self.speculative_config is not None
|
||||
set_eagle3_aux_hidden_state_layers(self.model, self.speculative_config)
|
||||
if isinstance(self.speculator, DraftModelSpeculator):
|
||||
self.speculator.load_model(self.model)
|
||||
eplb_models_added = self.eplb.maybe_register_speculator(
|
||||
self.speculator, self.speculative_config, load_dummy_weights
|
||||
)
|
||||
with use_workspace_lane(self._draft_workspace_lane):
|
||||
self.speculator.load_model(self.model)
|
||||
eplb_models_added = self.eplb.maybe_register_speculator(
|
||||
self.speculator, self.speculative_config, load_dummy_weights
|
||||
)
|
||||
time_after_load = time.perf_counter()
|
||||
|
||||
self.model_memory_usage = m.consumed_memory
|
||||
@@ -732,27 +737,28 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if hasattr(self.model, "get_mtp_target_hidden_states"):
|
||||
pre_hc_hidden_states = self.model.get_mtp_target_hidden_states()
|
||||
spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr]
|
||||
self.speculator.propose(
|
||||
input_batch=input_batch,
|
||||
attn_metadata=attn_metadata,
|
||||
slot_mappings=slot_mappings_by_layer,
|
||||
last_hidden_states=spec_hidden_states,
|
||||
aux_hidden_states=aux_hidden_states,
|
||||
num_sampled=torch.ones(
|
||||
input_batch.num_reqs, dtype=torch.int32, device=self.device
|
||||
),
|
||||
num_rejected=torch.zeros(
|
||||
input_batch.num_reqs, dtype=torch.int32, device=self.device
|
||||
),
|
||||
last_sampled=self.req_states.last_sampled_tokens,
|
||||
next_prefill_tokens=self.req_states.next_prefill_tokens,
|
||||
temperature=self.sampler.sampling_states.temperature.gpu,
|
||||
seeds=self.sampler.sampling_states.seeds.gpu,
|
||||
dummy_run=True,
|
||||
skip_attn_for_dummy_run=skip_attn,
|
||||
mm_inputs=mm_inputs,
|
||||
is_profile=is_profile,
|
||||
)
|
||||
with use_workspace_lane(self._draft_workspace_lane):
|
||||
self.speculator.propose(
|
||||
input_batch=input_batch,
|
||||
attn_metadata=attn_metadata,
|
||||
slot_mappings=slot_mappings_by_layer,
|
||||
last_hidden_states=spec_hidden_states,
|
||||
aux_hidden_states=aux_hidden_states,
|
||||
num_sampled=torch.ones(
|
||||
input_batch.num_reqs, dtype=torch.int32, device=self.device
|
||||
),
|
||||
num_rejected=torch.zeros(
|
||||
input_batch.num_reqs, dtype=torch.int32, device=self.device
|
||||
),
|
||||
last_sampled=self.req_states.last_sampled_tokens,
|
||||
next_prefill_tokens=self.req_states.next_prefill_tokens,
|
||||
temperature=self.sampler.sampling_states.temperature.gpu,
|
||||
seeds=self.sampler.sampling_states.seeds.gpu,
|
||||
dummy_run=True,
|
||||
skip_attn_for_dummy_run=skip_attn,
|
||||
mm_inputs=mm_inputs,
|
||||
is_profile=is_profile,
|
||||
)
|
||||
self.step_timing.drafter_end()
|
||||
|
||||
assert hidden_states is not None # Last PP rank always has hidden_states
|
||||
@@ -880,7 +886,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
lora_capture_hook=create_lora_capture_hook(self.lora_config, self),
|
||||
)
|
||||
if self.speculator is not None:
|
||||
self.speculator.capture()
|
||||
with use_workspace_lane(self._draft_workspace_lane):
|
||||
self.speculator.capture()
|
||||
if self.adaptive_verification is not None:
|
||||
with self.step_timing.collect() as timings:
|
||||
for batch in self.adaptive_verification.batches_to_profile(
|
||||
@@ -1799,20 +1806,21 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if hasattr(self.model, "get_mtp_target_hidden_states"):
|
||||
pre_hc_hidden_states = self.model.get_mtp_target_hidden_states()
|
||||
spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr]
|
||||
draft_tokens = self.speculator.propose(
|
||||
input_batch,
|
||||
attn_metadata,
|
||||
slot_mappings_by_layer,
|
||||
spec_hidden_states,
|
||||
aux_hidden_states,
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
self.req_states.last_sampled_tokens,
|
||||
self.req_states.next_prefill_tokens,
|
||||
self.sampler.sampling_states.temperature.gpu,
|
||||
self.sampler.sampling_states.seeds.gpu,
|
||||
mm_inputs=mm_inputs,
|
||||
)
|
||||
with use_workspace_lane(self._draft_workspace_lane):
|
||||
draft_tokens = self.speculator.propose(
|
||||
input_batch,
|
||||
attn_metadata,
|
||||
slot_mappings_by_layer,
|
||||
spec_hidden_states,
|
||||
aux_hidden_states,
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
self.req_states.last_sampled_tokens,
|
||||
self.req_states.next_prefill_tokens,
|
||||
self.sampler.sampling_states.temperature.gpu,
|
||||
self.sampler.sampling_states.seeds.gpu,
|
||||
mm_inputs=mm_inputs,
|
||||
)
|
||||
self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens
|
||||
if self.adaptive_verification is not None:
|
||||
self.adaptive_verification.record_confidences(
|
||||
|
||||
@@ -84,8 +84,11 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
self.sample_pos = torch.zeros(
|
||||
max_num_sampled_tokens, dtype=torch.int64, device=device
|
||||
)
|
||||
self.sample_idx_mapping = torch.zeros(
|
||||
max_num_sampled_tokens, dtype=torch.int32, device=device
|
||||
# -1 marks an inert sampling row. CUDA graph capture can execute the
|
||||
# full buffer before a real batch has populated it, so zero would make
|
||||
# every padding row scatter into request slot 0.
|
||||
self.sample_idx_mapping = torch.full(
|
||||
(max_num_sampled_tokens,), -1, dtype=torch.int32, device=device
|
||||
)
|
||||
# [0, 1, ..., N-1, 0, 1, ..., N-1, ...] -> the per-token column index into
|
||||
# draft_logits[req, step, :].
|
||||
@@ -255,7 +258,6 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
num_tokens_across_dp,
|
||||
cudagraph_runtime_mode,
|
||||
)
|
||||
|
||||
num_sample = num_reqs * self.num_speculative_steps
|
||||
sample_hidden_states = last_hidden_states[self.sample_indices[:num_sample]]
|
||||
# sample_pos is the predicted token's position Q; verification keys
|
||||
@@ -520,6 +522,7 @@ def _prepare_dflash_inputs_kernel(
|
||||
|
||||
num_rejected = tl.load(num_rejected_ptr + req_idx)
|
||||
valid_ctx_end = ctx_end - num_rejected
|
||||
num_valid_ctx = valid_ctx_end - ctx_start
|
||||
|
||||
num_sampled = tl.load(num_sampled_ptr + req_idx)
|
||||
if num_sampled > 0:
|
||||
@@ -533,20 +536,34 @@ def _prepare_dflash_inputs_kernel(
|
||||
|
||||
j = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
is_ctx = j < num_ctx
|
||||
is_query = (j >= num_ctx) & (j < num_ctx + num_query_per_req)
|
||||
query_off = j - num_ctx
|
||||
is_valid_ctx = j < num_valid_ctx
|
||||
is_query = (j >= num_valid_ctx) & (j < num_valid_ctx + num_query_per_req)
|
||||
query_off = j - num_valid_ctx
|
||||
|
||||
# --- Context positions / slots ---
|
||||
ctx_pos_idx = ctx_start + tl.where(is_ctx, j, 0)
|
||||
ctx_pos = tl.load(target_positions_ptr + ctx_pos_idx, mask=is_ctx, other=0)
|
||||
ctx_pos = tl.load(target_positions_ptr + ctx_pos_idx, mask=is_valid_ctx, other=0)
|
||||
ctx_block_num = ctx_pos // block_size
|
||||
ctx_block_num = tl.minimum(ctx_block_num, block_table_stride - 1)
|
||||
ctx_block_id = tl.load(
|
||||
block_table_ptr + req_idx * block_table_stride + ctx_block_num,
|
||||
mask=is_ctx,
|
||||
mask=is_valid_ctx,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
ctx_slot = ctx_block_id * block_size + (ctx_pos % block_size)
|
||||
# Block 0 is the null block. Old sliding-window context positions can map
|
||||
# to it after eviction; rejected suffix rows are invalid context as well.
|
||||
# Neither kind of row may write draft KV into physical block 0.
|
||||
ctx_resident = is_valid_ctx & (ctx_block_id != 0)
|
||||
ctx_slot = tl.where(
|
||||
ctx_resident,
|
||||
ctx_block_id * block_size + (ctx_pos % block_size),
|
||||
PAD_SLOT_ID,
|
||||
)
|
||||
# Stored over the full [0, num_ctx) span while the loads above are masked to
|
||||
# [0, num_valid_ctx): the rejected suffix rows in between get position 0 and
|
||||
# PAD_SLOT_ID. That is intentional — those rows write no KV and their
|
||||
# positions are never consumed, but the span must stay fully initialized so
|
||||
# a replayed graph cannot observe a stale value from an earlier batch.
|
||||
tl.store(out_context_positions_ptr + ctx_start + j, ctx_pos, mask=is_ctx)
|
||||
tl.store(out_context_slot_mapping_ptr + ctx_start + j, ctx_slot, mask=is_ctx)
|
||||
|
||||
@@ -563,7 +580,14 @@ def _prepare_dflash_inputs_kernel(
|
||||
mask=is_query,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
q_slot = q_block_id * block_size + (query_pos % block_size)
|
||||
# A null block is never a writable cache slot. This can occur when a
|
||||
# sliding-window block table contains evicted/global padding entries.
|
||||
q_resident = is_query & (q_block_id != 0)
|
||||
q_slot = tl.where(
|
||||
q_resident,
|
||||
q_block_id * block_size + (query_pos % block_size),
|
||||
PAD_SLOT_ID,
|
||||
)
|
||||
|
||||
tl.store(out_input_ids_ptr + query_idx, input_id, mask=is_query)
|
||||
clamped_query_pos = tl.minimum(query_pos, max_model_len - 1)
|
||||
|
||||
@@ -91,6 +91,16 @@ from .utils import request_memory
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _num_workspace_lanes(vllm_config: VllmConfig, use_v2_model_runner: bool) -> int:
|
||||
spec_config = vllm_config.speculative_config
|
||||
return (
|
||||
2
|
||||
if use_v2_model_runner and spec_config is not None and spec_config.use_dspark()
|
||||
else 1
|
||||
)
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.device_allocator.sleep_mode_backend import SleepModeBackend
|
||||
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
|
||||
@@ -402,9 +412,13 @@ class Worker(WorkerBase):
|
||||
else:
|
||||
raise RuntimeError(f"Unsupported device type: {self.device_config.device}")
|
||||
|
||||
# Initialize workspace manager
|
||||
# DSpark target and draft CUDA graphs retain workspace views concurrently.
|
||||
num_ubatches = 2 if self.vllm_config.parallel_config.enable_dbo else 1
|
||||
init_workspace_manager(self.device, num_ubatches)
|
||||
init_workspace_manager(
|
||||
self.device,
|
||||
num_ubatches,
|
||||
_num_workspace_lanes(self.vllm_config, self.use_v2_model_runner),
|
||||
)
|
||||
|
||||
# Construct the model runner
|
||||
if self.use_v2_model_runner:
|
||||
|
||||
+49
-14
@@ -3,6 +3,9 @@
|
||||
|
||||
import inspect
|
||||
import os
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from itertools import accumulate
|
||||
from math import prod
|
||||
|
||||
@@ -26,22 +29,43 @@ _GiB = 1024**3
|
||||
|
||||
# Global workspace manager instance
|
||||
_manager: "WorkspaceManager | None" = None
|
||||
_workspace_lane: ContextVar[int] = ContextVar("vllm_workspace_lane", default=0)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def use_workspace_lane(lane: int) -> Iterator[None]:
|
||||
"""Select an independent workspace owner for this execution context."""
|
||||
if lane < 0:
|
||||
raise ValueError(f"Workspace lane must be non-negative, got {lane}.")
|
||||
token = _workspace_lane.set(lane)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_workspace_lane.reset(token)
|
||||
|
||||
|
||||
class WorkspaceManager:
|
||||
"""Manager for workspace allocation.
|
||||
|
||||
Manages one workspace buffer per active ubatch slot.
|
||||
Manages one workspace buffer per active ``(ubatch, lane)`` slot.
|
||||
Can be locked to prevent further growth during execution.
|
||||
"""
|
||||
|
||||
def __init__(self, device: torch.device, num_ubatches: int | None = None):
|
||||
def __init__(
|
||||
self,
|
||||
device: torch.device,
|
||||
num_ubatches: int | None = None,
|
||||
num_lanes: int = 1,
|
||||
):
|
||||
self._device = device
|
||||
# Cache num ubatches at init based on configuration (default to 1)
|
||||
self._num_ubatches = num_ubatches if num_ubatches is not None else 1
|
||||
self._current_workspaces: list[torch.Tensor | None] = [
|
||||
None
|
||||
] * self._num_ubatches
|
||||
if num_lanes < 1:
|
||||
raise ValueError(f"num_lanes must be at least one, got {num_lanes}.")
|
||||
self._num_lanes = num_lanes
|
||||
self._current_workspaces: list[torch.Tensor | None] = [None] * (
|
||||
self._num_ubatches * self._num_lanes
|
||||
)
|
||||
self._locked: bool = False
|
||||
|
||||
@staticmethod
|
||||
@@ -126,7 +150,14 @@ class WorkspaceManager:
|
||||
The current workspace tensor.
|
||||
"""
|
||||
ubatch_id = dbo_current_ubatch_id()
|
||||
current_workspace = self._current_workspaces[ubatch_id]
|
||||
lane = _workspace_lane.get()
|
||||
if lane >= self._num_lanes:
|
||||
raise RuntimeError(
|
||||
f"Workspace lane {lane} is not configured; manager has "
|
||||
f"{self._num_lanes} lane(s)."
|
||||
)
|
||||
workspace_id = ubatch_id * self._num_lanes + lane
|
||||
current_workspace = self._current_workspaces[workspace_id]
|
||||
current_size = self._workspace_size_bytes(current_workspace)
|
||||
|
||||
if current_size < required_bytes:
|
||||
@@ -161,11 +192,11 @@ class WorkspaceManager:
|
||||
"Workspace growth is not allowed after locking."
|
||||
)
|
||||
|
||||
# Only resize the requesting ubatch's workspace. Other
|
||||
# ubatches resize lazily on their next get_simultaneous call.
|
||||
# Only resize the requesting ubatch/lane workspace. Other slots
|
||||
# resize lazily on their next get_simultaneous call.
|
||||
# Resizing all ubatches here would orphan the other ubatch's
|
||||
# old tensor when it still holds views into it (DBO leak).
|
||||
self._current_workspaces[ubatch_id] = None
|
||||
self._current_workspaces[workspace_id] = None
|
||||
del current_workspace
|
||||
# Release the freed segment back to CUDA so the caching
|
||||
# allocator can reuse the GPU memory for the larger
|
||||
@@ -173,19 +204,20 @@ class WorkspaceManager:
|
||||
# dead segment in reserved memory which can cause higher peak
|
||||
# memory usage.
|
||||
torch.accelerator.empty_cache()
|
||||
self._current_workspaces[ubatch_id] = torch.empty(
|
||||
self._current_workspaces[workspace_id] = torch.empty(
|
||||
(required_bytes,), dtype=torch.uint8, device=self._device
|
||||
)
|
||||
current_workspace = self._current_workspaces[ubatch_id]
|
||||
current_workspace = self._current_workspaces[workspace_id]
|
||||
|
||||
if envs.VLLM_DEBUG_WORKSPACE:
|
||||
logger.info(
|
||||
"[WORKSPACE DEBUG] Resized workspace from '%s': %.2f MB -> "
|
||||
"%.2f MB (ubatch %d)",
|
||||
"%.2f MB (ubatch %d, lane %d)",
|
||||
get_caller_info(),
|
||||
current_size / _MB,
|
||||
required_bytes / _MB,
|
||||
ubatch_id,
|
||||
lane,
|
||||
)
|
||||
|
||||
return current_workspace
|
||||
@@ -214,7 +246,9 @@ def current_workspace_manager() -> "WorkspaceManager":
|
||||
|
||||
|
||||
def init_workspace_manager(
|
||||
device: torch.device, num_ubatches: int | None = None
|
||||
device: torch.device,
|
||||
num_ubatches: int | None = None,
|
||||
num_lanes: int = 1,
|
||||
) -> None:
|
||||
"""Initialize the workspace manager with a device.
|
||||
|
||||
@@ -224,6 +258,7 @@ def init_workspace_manager(
|
||||
Args:
|
||||
device: The device to allocate workspace on.
|
||||
num_ubatches: Number of workspace ubatch slots. Defaults to 1.
|
||||
num_lanes: Number of independent execution lanes per ubatch. Defaults to 1.
|
||||
"""
|
||||
global _manager
|
||||
if _manager is not None:
|
||||
@@ -233,7 +268,7 @@ def init_workspace_manager(
|
||||
_manager._device,
|
||||
device,
|
||||
)
|
||||
_manager = WorkspaceManager(device, num_ubatches)
|
||||
_manager = WorkspaceManager(device, num_ubatches, num_lanes)
|
||||
|
||||
|
||||
def lock_workspace() -> None:
|
||||
|
||||
Reference in New Issue
Block a user