[Bugfix] Make DSV4 sparse MLA work end-to-end for plain decode, MTP, and DSpark (#51538)
This commit is contained in:
@@ -225,6 +225,7 @@ def test_flashinfer_sparse_indices_cache(monkeypatch):
|
||||
token_to_req_indices=torch.tensor([0, 1, 1], dtype=torch.int32),
|
||||
decode_swa_indices=torch.tensor([[5, 6, -1, -1]], dtype=torch.int32),
|
||||
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
|
||||
decode_swa_width=4,
|
||||
is_valid_token=torch.tensor([True], dtype=torch.bool),
|
||||
num_decodes=1,
|
||||
num_prefills=1,
|
||||
@@ -331,3 +332,120 @@ def test_flashinfer_sparse_indices_cache(monkeypatch):
|
||||
assert builder_calls == 4
|
||||
assert sparse_indices_third is not sparse_indices_fourth
|
||||
assert sparse_lens_third is not sparse_lens_fourth
|
||||
|
||||
|
||||
def test_flashinfer_sparse_index_preserves_logical_window(monkeypatch):
|
||||
from vllm.models.deepseek_v4.nvidia import flashinfer_sparse as flashinfer_mod
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
captured_shapes_and_windows: list[tuple[int, int]] = []
|
||||
|
||||
def fake_build(*args, **kwargs):
|
||||
# window_size is the 12th positional arg of
|
||||
# build_flashinfer_mixed_sparse_indices.
|
||||
captured_shapes_and_windows.append((args[0].shape[-1], args[11]))
|
||||
num_tokens = args[0].shape[0] + args[3].shape[0]
|
||||
return (
|
||||
torch.zeros((num_tokens, 1), dtype=torch.int32),
|
||||
torch.zeros((num_tokens,), dtype=torch.int32),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
flashinfer_mod, "build_flashinfer_mixed_sparse_indices", fake_build
|
||||
)
|
||||
|
||||
attn = object.__new__(flashinfer_mod.DeepseekV4FlashInferMLAAttention)
|
||||
attn.compress_ratio = 1
|
||||
attn.window_size = 4
|
||||
attn.topk_indices_buffer = torch.zeros((4, 0), dtype=torch.int32)
|
||||
|
||||
wide_width = 8
|
||||
wide_indices = torch.full((1, wide_width), -1, dtype=torch.int32)
|
||||
wide_indices[0, :2] = torch.tensor([5, 6], dtype=torch.int32)
|
||||
wide_metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
|
||||
slot_mapping=torch.tensor([0], dtype=torch.int64),
|
||||
block_size=64,
|
||||
seq_lens=torch.tensor([8], dtype=torch.int32),
|
||||
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
|
||||
query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.int32),
|
||||
token_to_req_indices=torch.tensor([0], dtype=torch.int32),
|
||||
decode_swa_indices=wide_indices,
|
||||
decode_swa_lens=torch.tensor([2], dtype=torch.int32),
|
||||
decode_swa_width=wide_width,
|
||||
is_valid_token=torch.tensor([True], dtype=torch.bool),
|
||||
num_decodes=1,
|
||||
num_prefills=0,
|
||||
num_decode_tokens=1,
|
||||
num_prefill_tokens=0,
|
||||
)
|
||||
attn._build_sparse_index_metadata(
|
||||
kv_cache=None,
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=wide_metadata,
|
||||
attn_metadata=None,
|
||||
swa_only=True,
|
||||
)
|
||||
assert captured_shapes_and_windows == [(wide_width, attn.window_size)]
|
||||
|
||||
empty_width = 8
|
||||
empty_metadata = DeepseekSparseSWAMetadata(
|
||||
block_table=torch.tensor([[0, 1]], dtype=torch.int32),
|
||||
slot_mapping=torch.tensor([0, 1], dtype=torch.int64),
|
||||
block_size=64,
|
||||
seq_lens=torch.tensor([8], dtype=torch.int32),
|
||||
query_start_loc=torch.tensor([0, 2], dtype=torch.int32),
|
||||
query_start_loc_cpu=torch.tensor([0, 2], dtype=torch.int32),
|
||||
token_to_req_indices=torch.tensor([0, 0], dtype=torch.int32),
|
||||
decode_swa_indices=torch.empty((0, 1, empty_width), dtype=torch.int32),
|
||||
decode_swa_lens=torch.empty((0,), dtype=torch.int32),
|
||||
decode_swa_width=empty_width,
|
||||
is_valid_token=torch.tensor([True, True], dtype=torch.bool),
|
||||
num_decodes=0,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=0,
|
||||
num_prefill_tokens=2,
|
||||
)
|
||||
attn._build_sparse_index_metadata(
|
||||
kv_cache=None,
|
||||
swa_k_cache=torch.empty((1, 64, 512), dtype=torch.bfloat16),
|
||||
swa_metadata=empty_metadata,
|
||||
attn_metadata=None,
|
||||
swa_only=True,
|
||||
)
|
||||
assert captured_shapes_and_windows == [
|
||||
(wide_width, attn.window_size),
|
||||
(empty_width, attn.window_size),
|
||||
]
|
||||
|
||||
|
||||
def test_flashinfer_mixed_sparse_indices_separates_window_and_padded_width():
|
||||
from vllm.models.deepseek_v4.common.ops.cache_utils import (
|
||||
build_flashinfer_mixed_sparse_indices,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
padded_width = 8
|
||||
logical_window = 4
|
||||
sparse_indices, sparse_lens = build_flashinfer_mixed_sparse_indices(
|
||||
decode_swa_indices=torch.empty(
|
||||
(0, padded_width), dtype=torch.int32, device=device
|
||||
),
|
||||
decode_compressed_indices=None,
|
||||
decode_compressed_topk_lens=None,
|
||||
prefill_topk_indices=torch.empty((1, 0), dtype=torch.int32, device=device),
|
||||
query_start_loc=torch.tensor([0, 1], dtype=torch.int32, device=device),
|
||||
seq_lens=torch.tensor([logical_window], dtype=torch.int32, device=device),
|
||||
token_to_req_indices=torch.tensor([0], dtype=torch.int32, device=device),
|
||||
swa_block_table=torch.tensor([[0]], dtype=torch.int32, device=device),
|
||||
swa_block_size=64,
|
||||
compressed_block_table=None,
|
||||
compressed_block_size=64,
|
||||
window_size=logical_window,
|
||||
compress_ratio=1,
|
||||
topk=0,
|
||||
)
|
||||
|
||||
assert sparse_indices.shape == (1, padded_width)
|
||||
assert sparse_indices[0].cpu().tolist() == [0, 1, 2, 3, -1, -1, -1, -1]
|
||||
assert sparse_lens.cpu().tolist() == [padded_width]
|
||||
|
||||
@@ -1095,6 +1095,101 @@ def test_convert_standard_mxfp4_weights_for_flashinfer_cutlass(
|
||||
torch.testing.assert_close(interleaved_scales[1], w2_scale)
|
||||
|
||||
|
||||
def test_gpt_oss_quant_config_supplies_clamped_swiglu_params():
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
|
||||
from vllm.model_executor.layers.quantization.mxfp4 import GptOssMxfp4MoEMethod
|
||||
|
||||
fake_method = types.SimpleNamespace(
|
||||
mxfp4_backend=Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
||||
w13_precision_config=None,
|
||||
w2_precision_config=None,
|
||||
)
|
||||
fake_layer = types.SimpleNamespace(
|
||||
w13_weight_scale=torch.zeros(2, 4, 2, dtype=torch.uint8),
|
||||
w2_weight_scale=torch.zeros(2, 4, 2, dtype=torch.uint8),
|
||||
)
|
||||
|
||||
quant_config = GptOssMxfp4MoEMethod.get_fused_moe_quant_config(
|
||||
fake_method, fake_layer
|
||||
)
|
||||
|
||||
assert quant_config is not None
|
||||
assert quant_config.gemm1_alpha == 1.702
|
||||
assert quant_config.gemm1_beta == 1.0
|
||||
assert quant_config.gemm1_clamp_limit == 7.0
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="CUDA is required for FlashInferExperts parameter storage",
|
||||
)
|
||||
@pytest.mark.parametrize("supplied", [False, True])
|
||||
def test_flashinfer_experts_swiglu_params_follow_quant_config(supplied: bool):
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
mxfp4_w4a16_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
|
||||
FlashInferExperts,
|
||||
)
|
||||
|
||||
num_experts, intermediate_size, hidden_size = 2, 128, 256
|
||||
w1_scale = torch.zeros(
|
||||
(num_experts, 2 * intermediate_size, hidden_size // 32),
|
||||
dtype=torch.uint8,
|
||||
device="cuda",
|
||||
)
|
||||
w2_scale = torch.zeros(
|
||||
(num_experts, hidden_size, intermediate_size // 32),
|
||||
dtype=torch.uint8,
|
||||
device="cuda",
|
||||
)
|
||||
extra = (
|
||||
{"gemm1_alpha": 0.5, "gemm1_beta": 0.25, "gemm1_clamp_limit": 3.0}
|
||||
if supplied
|
||||
else {}
|
||||
)
|
||||
quant_config = mxfp4_w4a16_moe_quant_config(
|
||||
w1_scale=w1_scale, w2_scale=w2_scale, **extra
|
||||
)
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=num_experts,
|
||||
experts_per_token=1,
|
||||
hidden_dim=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
num_local_experts=num_experts,
|
||||
num_logical_experts=num_experts,
|
||||
activation=MoEActivation.SILU,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=torch.bfloat16,
|
||||
)
|
||||
experts = FlashInferExperts(moe_config=moe_config, quant_config=quant_config)
|
||||
|
||||
if supplied:
|
||||
for name, value in (
|
||||
("gemm1_alpha", 0.5),
|
||||
("gemm1_beta", 0.25),
|
||||
("gemm1_clamp_limit", 3.0),
|
||||
):
|
||||
parameter = getattr(experts, name)
|
||||
assert isinstance(parameter, torch.Tensor)
|
||||
assert parameter.dtype == torch.float32
|
||||
assert parameter.shape == (num_experts,)
|
||||
torch.testing.assert_close(
|
||||
parameter.cpu(),
|
||||
torch.full((num_experts,), value, dtype=torch.float32),
|
||||
)
|
||||
else:
|
||||
assert experts.gemm1_alpha is None
|
||||
assert experts.gemm1_beta is None
|
||||
assert experts.gemm1_clamp_limit is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("topk", [1, 4])
|
||||
@pytest.mark.parametrize("num_experts", [32])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 128])
|
||||
|
||||
@@ -31,10 +31,23 @@ from vllm.models.deepseek_v4.common.ops.fused_compress_quant_cache import (
|
||||
)
|
||||
from vllm.models.deepseek_v4.compressor import _get_c128_boundary
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
get_dspark_swa_index_width,
|
||||
)
|
||||
|
||||
from .test_fused_indexer_q_rope_quant import quantize_to_mxfp4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("window_size", "num_speculative_tokens", "expected"),
|
||||
[(128, 5, 192), (512, 5, 576), (1024, 0, 1024)],
|
||||
)
|
||||
def test_get_dspark_swa_index_width(
|
||||
window_size: int, num_speculative_tokens: int, expected: int
|
||||
):
|
||||
assert get_dspark_swa_index_width(window_size, num_speculative_tokens) == expected
|
||||
|
||||
|
||||
def test_compute_global_topk_reuses_output_buffers():
|
||||
device = "cuda"
|
||||
topk_indices = torch.tensor(
|
||||
|
||||
@@ -7,6 +7,9 @@ from types import SimpleNamespace
|
||||
import torch
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.models.deepseek_v4.nvidia.flashinfer_sparse import (
|
||||
_required_sm120_sparse_topk,
|
||||
)
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.utils import flashinfer as fi_utils
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
@@ -52,3 +55,33 @@ def test_v32_glm_sm120_backend_accepts_glm_block_size(
|
||||
)
|
||||
|
||||
assert invalid_reasons == []
|
||||
|
||||
|
||||
def test_sm120_dsv4_capability_checks_exact_dispatch_shape(monkeypatch) -> None:
|
||||
fake_module = SimpleNamespace(
|
||||
_DECODE_DSV4_DISPATCH=frozenset({(32, 128), (32, 192)})
|
||||
)
|
||||
monkeypatch.setattr(fi_utils, "has_flashinfer_sparse_mla_sm120", lambda: True)
|
||||
monkeypatch.setattr(fi_utils, "_get_submodule", lambda _name: fake_module)
|
||||
fi_utils.has_flashinfer_sparse_mla_sm120_config.cache_clear()
|
||||
|
||||
assert fi_utils.has_flashinfer_sparse_mla_sm120_config(32, 128)
|
||||
assert fi_utils.has_flashinfer_sparse_mla_sm120_config(32, 192)
|
||||
assert not fi_utils.has_flashinfer_sparse_mla_sm120_config(32, 256)
|
||||
assert not fi_utils.has_flashinfer_sparse_mla_sm120_config(16, 192)
|
||||
|
||||
fi_utils.has_flashinfer_sparse_mla_sm120_config.cache_clear()
|
||||
|
||||
|
||||
def test_sm120_dsv4_required_topk_tracks_dspark_width() -> None:
|
||||
causal = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(use_non_causal=False),
|
||||
speculative_config=SimpleNamespace(num_speculative_tokens=5),
|
||||
)
|
||||
dspark = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(use_non_causal=True),
|
||||
speculative_config=SimpleNamespace(num_speculative_tokens=5),
|
||||
)
|
||||
|
||||
assert _required_sm120_sparse_topk(causal, 128) == 128
|
||||
assert _required_sm120_sparse_topk(dspark, 128) == 192
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import (
|
||||
prepare_dflash_inputs,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="requires a CUDA device"
|
||||
)
|
||||
|
||||
|
||||
def _run_prepare(*, target_positions: list[int], block_table_values: list[int]):
|
||||
device = torch.device("cuda")
|
||||
max_num_reqs = 4
|
||||
max_num_tokens = 16
|
||||
num_speculative_steps = 3
|
||||
|
||||
input_buffers = SimpleNamespace(
|
||||
input_ids=torch.full((max_num_tokens,), -1, dtype=torch.int32, device=device),
|
||||
positions=torch.full((max_num_tokens,), -1, dtype=torch.int64, device=device),
|
||||
query_start_loc=torch.full(
|
||||
(max_num_reqs + 1,), -1, dtype=torch.int32, device=device
|
||||
),
|
||||
seq_lens=torch.full((max_num_reqs,), -1, dtype=torch.int32, device=device),
|
||||
)
|
||||
input_batch = SimpleNamespace(
|
||||
num_reqs=1,
|
||||
num_scheduled_tokens=np.array([4], dtype=np.int32),
|
||||
positions=torch.tensor(target_positions, dtype=torch.int64, device=device),
|
||||
query_start_loc=torch.tensor([0, 4], dtype=torch.int32, device=device),
|
||||
idx_mapping=torch.tensor([2], dtype=torch.int32, device=device),
|
||||
)
|
||||
query_slot_mapping = torch.full(
|
||||
(max_num_tokens,), -2, dtype=torch.int64, device=device
|
||||
)
|
||||
context_positions = torch.full(
|
||||
(max_num_tokens,), -1, dtype=torch.int64, device=device
|
||||
)
|
||||
context_slot_mapping = torch.full(
|
||||
(max_num_tokens,), -2, dtype=torch.int64, device=device
|
||||
)
|
||||
sample_indices = torch.full(
|
||||
(max_num_reqs * num_speculative_steps,),
|
||||
-1,
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
sample_pos = torch.full_like(sample_indices, -1)
|
||||
sample_idx_mapping = torch.full(
|
||||
sample_indices.shape, -1, dtype=torch.int32, device=device
|
||||
)
|
||||
temperature = torch.zeros(max_num_reqs, dtype=torch.float32, device=device)
|
||||
seeds = torch.zeros(max_num_reqs, dtype=torch.int64, device=device)
|
||||
input_temperature = torch.tensor(
|
||||
[0.0, 0.0, 1.0, 0.0], dtype=torch.float32, device=device
|
||||
)
|
||||
input_seeds = torch.tensor([0, 0, 17, 0], dtype=torch.int64, device=device)
|
||||
last_sampled = torch.tensor([0, 0, 99, 0], dtype=torch.int64, device=device)
|
||||
next_prefill_tokens = torch.zeros_like(last_sampled)
|
||||
block_table = torch.tensor([block_table_values], dtype=torch.int32, device=device)
|
||||
|
||||
prepare_dflash_inputs(
|
||||
input_buffers,
|
||||
query_slot_mapping,
|
||||
context_positions,
|
||||
context_slot_mapping,
|
||||
sample_indices,
|
||||
sample_pos,
|
||||
sample_idx_mapping,
|
||||
temperature,
|
||||
seeds,
|
||||
input_batch,
|
||||
torch.tensor([1], dtype=torch.int32, device=device),
|
||||
torch.tensor([2], dtype=torch.int32, device=device),
|
||||
last_sampled,
|
||||
next_prefill_tokens,
|
||||
input_temperature,
|
||||
input_seeds,
|
||||
block_table,
|
||||
4,
|
||||
123,
|
||||
num_speculative_steps,
|
||||
num_speculative_steps,
|
||||
max_num_reqs,
|
||||
max_num_tokens,
|
||||
128,
|
||||
sample_from_anchor=True,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
return SimpleNamespace(
|
||||
input_buffers=input_buffers,
|
||||
query_slot_mapping=query_slot_mapping.cpu(),
|
||||
context_positions=context_positions.cpu(),
|
||||
context_slot_mapping=context_slot_mapping.cpu(),
|
||||
sample_indices=sample_indices.cpu(),
|
||||
sample_pos=sample_pos.cpu(),
|
||||
sample_idx_mapping=sample_idx_mapping.cpu(),
|
||||
temperature=temperature.cpu(),
|
||||
seeds=seeds.cpu(),
|
||||
)
|
||||
|
||||
|
||||
def test_prepare_dflash_inputs_excludes_rejected_context_suffix():
|
||||
# Positions 10/11 use physical block 7. Rejected positions 12/13 would use
|
||||
# block 8, but must be PAD context rather than contaminating draft KV.
|
||||
out = _run_prepare(
|
||||
target_positions=[10, 11, 12, 13],
|
||||
block_table_values=[0, 0, 7, 8, 9, 10, 11, 12],
|
||||
)
|
||||
|
||||
assert out.context_positions[:4].tolist() == [10, 11, 0, 0]
|
||||
assert out.context_slot_mapping[:4].tolist() == [30, 31, PAD_SLOT_ID, PAD_SLOT_ID]
|
||||
|
||||
# The replacement query starts immediately after the two valid rows and
|
||||
# advances from the last accepted position (11).
|
||||
assert out.input_buffers.input_ids[:3].cpu().tolist() == [99, 123, 123]
|
||||
assert out.input_buffers.positions[:3].cpu().tolist() == [12, 13, 14]
|
||||
assert out.query_slot_mapping[:3].tolist() == [32, 33, 34]
|
||||
assert out.sample_indices[:3].tolist() == [0, 1, 2]
|
||||
assert out.sample_pos[:3].tolist() == [13, 14, 15]
|
||||
assert out.sample_idx_mapping[:3].tolist() == [2, 2, 2]
|
||||
assert out.temperature[2].item() == 1.0
|
||||
assert out.seeds[2].item() == 17
|
||||
|
||||
|
||||
def test_prepare_dflash_inputs_never_writes_the_null_block():
|
||||
# The valid context uses logical block 0 and the replacement query uses
|
||||
# logical block 1. Both map to the null block and must remain unwritable.
|
||||
out = _run_prepare(
|
||||
target_positions=[2, 3, 4, 5],
|
||||
block_table_values=[0, 0, 7, 8, 9, 10, 11, 12],
|
||||
)
|
||||
|
||||
assert out.context_slot_mapping[:4].tolist() == [
|
||||
PAD_SLOT_ID,
|
||||
PAD_SLOT_ID,
|
||||
PAD_SLOT_ID,
|
||||
PAD_SLOT_ID,
|
||||
]
|
||||
assert out.query_slot_mapping[:3].tolist() == [
|
||||
PAD_SLOT_ID,
|
||||
PAD_SLOT_ID,
|
||||
PAD_SLOT_ID,
|
||||
]
|
||||
@@ -0,0 +1,102 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.worker.workspace as workspace
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.worker.gpu_worker import _num_workspace_lanes
|
||||
|
||||
|
||||
class _SpecConfig:
|
||||
def __init__(self, dspark: bool) -> None:
|
||||
self._dspark = dspark
|
||||
|
||||
def use_dspark(self) -> bool:
|
||||
return self._dspark
|
||||
|
||||
|
||||
class _VllmConfig:
|
||||
def __init__(self, spec_config: _SpecConfig | None) -> None:
|
||||
self.speculative_config = spec_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("use_v2_model_runner", "spec_config", "expected"),
|
||||
[
|
||||
(True, _SpecConfig(True), 2),
|
||||
(False, _SpecConfig(True), 1),
|
||||
(True, _SpecConfig(False), 1),
|
||||
(True, None, 1),
|
||||
],
|
||||
)
|
||||
def test_workspace_lane_count_is_dspark_only(
|
||||
use_v2_model_runner: bool,
|
||||
spec_config: _SpecConfig | None,
|
||||
expected: int,
|
||||
) -> None:
|
||||
config = cast(VllmConfig, _VllmConfig(spec_config))
|
||||
assert _num_workspace_lanes(config, use_v2_model_runner) == expected
|
||||
|
||||
|
||||
def test_workspace_lanes_do_not_alias_and_restore_context(monkeypatch) -> None:
|
||||
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: 0)
|
||||
manager = workspace.WorkspaceManager(
|
||||
torch.device("cpu"), num_ubatches=2, num_lanes=2
|
||||
)
|
||||
|
||||
assert manager._current_workspaces == [None, None, None, None]
|
||||
|
||||
(target,) = manager.get_simultaneous(((512,), torch.uint8))
|
||||
with workspace.use_workspace_lane(1):
|
||||
(draft,) = manager.get_simultaneous(((256,), torch.uint8))
|
||||
(draft_reused,) = manager.get_simultaneous(((8,), torch.uint8))
|
||||
(target_reused,) = manager.get_simultaneous(((8,), torch.uint8))
|
||||
|
||||
assert manager._current_workspaces[0].numel() == 512 # type: ignore[union-attr]
|
||||
assert manager._current_workspaces[1].numel() == 256 # type: ignore[union-attr]
|
||||
assert manager._current_workspaces[2:] == [None, None]
|
||||
assert target.data_ptr() != draft.data_ptr()
|
||||
assert draft.data_ptr() == draft_reused.data_ptr()
|
||||
assert target.data_ptr() == target_reused.data_ptr()
|
||||
|
||||
|
||||
def test_workspace_lanes_compose_with_ubatches(monkeypatch) -> None:
|
||||
active_ubatch = [0]
|
||||
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: active_ubatch[0])
|
||||
manager = workspace.WorkspaceManager(
|
||||
torch.device("cpu"), num_ubatches=2, num_lanes=2
|
||||
)
|
||||
|
||||
pointers = set()
|
||||
for ubatch_id in range(2):
|
||||
active_ubatch[0] = ubatch_id
|
||||
for lane in range(2):
|
||||
with workspace.use_workspace_lane(lane):
|
||||
(buffer,) = manager.get_simultaneous(((16,), torch.uint8))
|
||||
pointers.add(buffer.data_ptr())
|
||||
|
||||
assert len(pointers) == 4
|
||||
|
||||
|
||||
def test_workspace_lane_validation(monkeypatch) -> None:
|
||||
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: 0)
|
||||
manager = workspace.WorkspaceManager(torch.device("cpu"), num_lanes=1)
|
||||
|
||||
with (
|
||||
pytest.raises(ValueError, match="non-negative"),
|
||||
workspace.use_workspace_lane(-1),
|
||||
):
|
||||
pass
|
||||
|
||||
with (
|
||||
workspace.use_workspace_lane(1),
|
||||
pytest.raises(RuntimeError, match="is not configured"),
|
||||
):
|
||||
manager.get_simultaneous(((1,), torch.uint8))
|
||||
|
||||
with pytest.raises(ValueError, match="at least one"):
|
||||
workspace.WorkspaceManager(torch.device("cpu"), num_lanes=0)
|
||||
Reference in New Issue
Block a user