Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -763,7 +763,7 @@ def test_flashinfer_cross_layer_layout(
|
||||
|
||||
_test_backend_correctness(
|
||||
batch_spec=BATCH_SPECS[batch_spec_name],
|
||||
model="meta-llama/Meta-Llama-3-8B",
|
||||
model="Qwen/Qwen3-0.6B",
|
||||
backend_to_test=[AttentionBackendEnum.FLASHINFER],
|
||||
mask_mod=causal_mask_mod,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
@@ -786,7 +786,7 @@ def test_flashinfer_cross_layer_layout(
|
||||
"single_prefill",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model", ["meta-llama/Meta-Llama-3-8B"])
|
||||
@pytest.mark.parametrize("model", ["Qwen/Qwen3-0.6B"])
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e4m3"])
|
||||
def test_causal_backend_correctness(
|
||||
@@ -1582,7 +1582,7 @@ def test_flashinfer_native_prefill_with_sinks(default_vllm_config):
|
||||
|
||||
_test_backend_correctness(
|
||||
BATCH_SPECS["small_prefill"],
|
||||
"meta-llama/Meta-Llama-3-8B",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
[AttentionBackendEnum.FLASHINFER],
|
||||
causal_mask_mod,
|
||||
use_sinks=True,
|
||||
@@ -1629,7 +1629,7 @@ def test_flashinfer_xqa_decode_correctness(default_vllm_config):
|
||||
|
||||
batch_spec = BATCH_SPECS["small_decode"]
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="meta-llama/Meta-Llama-3-8B",
|
||||
model_name="Qwen/Qwen3-0.6B",
|
||||
max_model_len=max(batch_spec.seq_lens),
|
||||
block_size=16,
|
||||
)
|
||||
@@ -1691,7 +1691,7 @@ def test_flashinfer_xqa_decode_correctness(default_vllm_config):
|
||||
|
||||
_test_backend_correctness(
|
||||
batch_spec,
|
||||
"meta-llama/Meta-Llama-3-8B",
|
||||
"Qwen/Qwen3-0.6B",
|
||||
[AttentionBackendEnum.FLASHINFER],
|
||||
causal_mask_mod,
|
||||
)
|
||||
@@ -1994,7 +1994,7 @@ if current_platform.is_rocm():
|
||||
"mixed_small",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model", ["meta-llama/Meta-Llama-3-8B"])
|
||||
@pytest.mark.parametrize("model", ["Qwen/Qwen3-0.6B"])
|
||||
def test_non_causal_backend_correctness(
|
||||
default_vllm_config,
|
||||
batch_spec_name: str,
|
||||
|
||||
@@ -67,7 +67,7 @@ def test_deepseek_rocm_adaptive_builders_support_varlen_full_graphs():
|
||||
DeepseekV4ROCMAiterMLASparseMetadataBuilder,
|
||||
DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
|
||||
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
|
||||
v41_rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
|
||||
v41_rocm.DeepseekV41ROCMAiterSparseSWAMetadataBuilder,
|
||||
):
|
||||
assert (
|
||||
builder_cls.get_cudagraph_support(adaptive_config, SimpleNamespace())
|
||||
@@ -126,7 +126,7 @@ def test_deepseek_v41_rocm_fixed_k_path_is_unchanged(
|
||||
|
||||
for builder_cls in (
|
||||
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
|
||||
v41_rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
|
||||
v41_rocm.DeepseekV41ROCMAiterSparseSWAMetadataBuilder,
|
||||
):
|
||||
assert (
|
||||
builder_cls.get_cudagraph_support(fixed_config, SimpleNamespace())
|
||||
|
||||
@@ -529,7 +529,7 @@ def make_builder(
|
||||
window: int = WINDOW,
|
||||
max_image_tokens: int = MAX_IMG,
|
||||
max_num_batched_tokens: int = 64,
|
||||
model_name: str = "meta-llama/Meta-Llama-3-8B",
|
||||
model_name: str = "Qwen/Qwen3-0.6B",
|
||||
builder_cls: type[
|
||||
DeepseekSparseSWAMetadataBuilder
|
||||
] = DeepseekSparseSWAMetadataBuilder,
|
||||
|
||||
@@ -10,9 +10,11 @@ import torch
|
||||
|
||||
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
|
||||
import vllm.v1.attention.backends.mla.indexer as indexer
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.config import AttentionConfig, CUDAGraphMode
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import PrefillRowShard
|
||||
|
||||
MIN_ROWS = AttentionConfig().indexer_tp_shard_min_rows
|
||||
|
||||
|
||||
def _chunk(start: int, end: int) -> SimpleNamespace:
|
||||
"""Chunk over rows [start, end); bounds derive from the global row."""
|
||||
@@ -80,11 +82,11 @@ def test_shard_exchange_reassembles_rows_only(
|
||||
assert torch.all(buffer[decode + sum(sizes) :] == -7), "padding untouched"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2, 4])
|
||||
@pytest.mark.parametrize("tp_size", [2, 4, 8])
|
||||
def test_balanced_row_shard_partitions_and_declines(tp_size: int) -> None:
|
||||
"""The split covers every row once with >= 1 row per rank and declines
|
||||
below the activation floor (or on a single rank)."""
|
||||
n = indexer.MIN_TP_SHARD_ROWS_PER_RANK * tp_size
|
||||
n = MIN_ROWS
|
||||
shapes = [
|
||||
([4 * n], [4 * n]), # fresh prompt: the full causal cost ramp
|
||||
# uneven requests at different context depths
|
||||
@@ -97,12 +99,38 @@ def test_balanced_row_shard_partitions_and_declines(tp_size: int) -> None:
|
||||
torch.tensor(query_lens, dtype=torch.int32),
|
||||
ratio,
|
||||
tp_size,
|
||||
n,
|
||||
)
|
||||
assert sizes is not None and len(sizes) == tp_size
|
||||
assert sum(sizes) == sum(query_lens) and min(sizes) >= 1
|
||||
lens = torch.tensor([n - 1], dtype=torch.int32)
|
||||
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, tp_size) is None
|
||||
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, 1) is None
|
||||
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, tp_size, n) is None
|
||||
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, 1, n) is None
|
||||
|
||||
|
||||
def test_row_shard_threshold_is_total_not_per_rank() -> None:
|
||||
"""The activation floor counts TOTAL rows regardless of tp_size: a
|
||||
per-rank floor would need 128K rows at TP8, i.e. an unschedulable
|
||||
--max-num-batched-tokens of 128K on 80 GB Hopper nodes."""
|
||||
lens = torch.tensor([MIN_ROWS], dtype=torch.int32)
|
||||
for tp_size in (2, 4, 8):
|
||||
sizes = indexer.balanced_prefill_row_shard(lens, lens, 4, tp_size, MIN_ROWS)
|
||||
assert sizes is not None
|
||||
|
||||
|
||||
def test_tp2_old_per_rank_window_stays_replicated() -> None:
|
||||
"""32K-64K total rows engaged on TP2 under the old per-rank floor
|
||||
(16K rows/rank); the total floor now keeps them replicated — the
|
||||
all_gatherv latency term is largest at small tp."""
|
||||
window = torch.tensor([3 * MIN_ROWS // 4], dtype=torch.int32)
|
||||
assert indexer.balanced_prefill_row_shard(window, window, 4, 2, MIN_ROWS) is None
|
||||
|
||||
|
||||
def test_row_shard_floor_is_at_least_tp_size() -> None:
|
||||
"""A floor below tp_size still leaves every rank at least one row."""
|
||||
lens = torch.tensor([8], dtype=torch.int32)
|
||||
assert indexer.balanced_prefill_row_shard(lens, lens, 4, 8, 1) is not None
|
||||
assert indexer.balanced_prefill_row_shard(lens - 1, lens - 1, 4, 8, 1) is None
|
||||
|
||||
|
||||
def test_row_sharding_gate_rejects_unsupported_configurations(
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Equivalence tests for the fused mamba "align" block-table gather.
|
||||
|
||||
`mamba_get_block_table_tensor`'s align branch used to be five PyTorch
|
||||
statements that expanded to seven kernel launches over a `(num_reqs,)` tensor.
|
||||
It is now one Triton kernel. These tests pin the fused kernel to the exact
|
||||
semantics of the expression it replaced, including cases a single-request
|
||||
decode never exercises but other callers can hit.
|
||||
|
||||
`block_table` is deliberately tested non-contiguous: the original
|
||||
`torch.gather(block_table, 1, ...)` honours any layout, and this is a shared
|
||||
utility (`gdn_attn`, `mamba_attn`, `short_conv_attn` and `linear_attn` all call
|
||||
it), so a column stride of 1 must not be assumed.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.backends.utils import mamba_get_block_table_tensor
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
WIDTH = 64
|
||||
|
||||
|
||||
def _spec(block_size: int, num_speculative_blocks: int) -> MambaSpec:
|
||||
# Only block_size and num_speculative_blocks are read by the align branch;
|
||||
# shapes/dtypes just have to be well formed for the frozen dataclass.
|
||||
return MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1,),),
|
||||
dtypes=(torch.float32,),
|
||||
num_speculative_blocks=num_speculative_blocks,
|
||||
)
|
||||
|
||||
|
||||
def _reference(block_table, seq_lens, block_size, num_spec_blocks):
|
||||
"""The align branch exactly as it was before the fusion."""
|
||||
start_indices = (seq_lens - 1) // block_size
|
||||
start_indices = start_indices.clamp(min=0)
|
||||
offsets = torch.arange(
|
||||
1 + num_spec_blocks, device=block_table.device, dtype=torch.int32
|
||||
)
|
||||
indices_to_gather = (start_indices.unsqueeze(1) + offsets).to(torch.int64)
|
||||
return torch.gather(block_table, 1, indices_to_gather)
|
||||
|
||||
|
||||
def _make(nreq, block_size, nspec, device, noncontig=False):
|
||||
torch.manual_seed(0)
|
||||
width = WIDTH * 2 if noncontig else WIDTH
|
||||
bt = torch.randint(0, 5000, (nreq, width), dtype=torch.int32, device=device)
|
||||
if noncontig:
|
||||
# Every other column: stride(1) == 2, which a kernel assuming a unit
|
||||
# column stride would read straight through, silently returning the
|
||||
# wrong blocks.
|
||||
bt = bt[:, ::2]
|
||||
assert not bt.is_contiguous()
|
||||
# Keep start + nspec inside the row, which the gather already requires.
|
||||
hi = max(1, (WIDTH - 1 - nspec) * block_size)
|
||||
sl = torch.randint(0, hi, (nreq,), dtype=torch.int32, device=device)
|
||||
return bt, sl
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
|
||||
@pytest.mark.parametrize("nreq", [1, 2, 8, 64])
|
||||
@pytest.mark.parametrize("block_size", [16, 64, 256])
|
||||
@pytest.mark.parametrize("nspec", [0, 1, 2, 3, 4, 7])
|
||||
def test_matches_reference(nreq, block_size, nspec):
|
||||
bt, sl = _make(nreq, block_size, nspec, "cuda")
|
||||
got = mamba_get_block_table_tensor(bt, sl, _spec(block_size, nspec), "align")
|
||||
want = _reference(bt, sl, block_size, nspec)
|
||||
assert got.shape == want.shape
|
||||
assert got.dtype == want.dtype
|
||||
torch.testing.assert_close(got, want, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
|
||||
@pytest.mark.parametrize("nspec", [0, 3, 7])
|
||||
def test_non_contiguous_block_table(nspec):
|
||||
"""A column stride != 1 must be honoured, not assumed away."""
|
||||
bt, sl = _make(8, 64, nspec, "cuda", noncontig=True)
|
||||
got = mamba_get_block_table_tensor(bt, sl, _spec(64, nspec), "align")
|
||||
want = _reference(bt, sl, 64, nspec)
|
||||
torch.testing.assert_close(got, want, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
|
||||
def test_zero_seq_len_clamps_to_first_row():
|
||||
"""CUDA-graph padding rows carry seq_len 0; the old code clamped to row 0."""
|
||||
bt = torch.randint(0, 5000, (4, WIDTH), dtype=torch.int32, device="cuda")
|
||||
sl = torch.zeros(4, dtype=torch.int32, device="cuda")
|
||||
got = mamba_get_block_table_tensor(bt, sl, _spec(64, 3), "align")
|
||||
torch.testing.assert_close(got, _reference(bt, sl, 64, 3), atol=0, rtol=0)
|
||||
# start clamps to 0, so this is just the first 1 + nspec columns.
|
||||
torch.testing.assert_close(got, bt[:, :4], atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
|
||||
def test_zero_requests_returns_empty():
|
||||
"""An empty batch must not launch a (0,) grid."""
|
||||
bt = torch.zeros((0, WIDTH), dtype=torch.int32, device="cuda")
|
||||
sl = torch.zeros(0, dtype=torch.int32, device="cuda")
|
||||
got = mamba_get_block_table_tensor(bt, sl, _spec(64, 3), "align")
|
||||
assert got.shape == (0, 4)
|
||||
assert got.dtype == bt.dtype
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
|
||||
def test_none_mode_returns_the_input_unchanged():
|
||||
"""`MambaCacheMode` is Literal["align", "none"]; only "none" passes through.
|
||||
|
||||
An earlier version of this test also parametrised a "mode" of "all", which
|
||||
is not a member of that Literal. Anything that is not "none" takes the
|
||||
align branch and returns a NEW tensor, so asserting identity for it failed
|
||||
-- correctly. The passthrough contract applies to "none" alone.
|
||||
"""
|
||||
bt = torch.randint(0, 5000, (4, WIDTH), dtype=torch.int32, device="cuda")
|
||||
sl = torch.randint(0, 1000, (4,), dtype=torch.int32, device="cuda")
|
||||
assert mamba_get_block_table_tensor(bt, sl, _spec(64, 3), "none") is bt
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CPU tensors. Every test above runs on "cuda", which is exactly how the
|
||||
# regression reached review: `mamba_get_block_table_tensor` is a SHARED
|
||||
# utility and its callers legitimately pass CPU tensors -- the GDN and
|
||||
# linear-attention metadata builders are unit-tested with
|
||||
# `DEVICE = torch.device("cpu")`. A Triton kernel cannot run on those at all,
|
||||
# while the torch expression it replaced could. These run without a GPU.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("nspec", [0, 1, 3])
|
||||
@pytest.mark.parametrize("block_size", [1, 16, 64])
|
||||
def test_cpu_tensors_match_the_reference(nspec, block_size):
|
||||
"""The align branch must work on CPU, not just on the accelerator."""
|
||||
width = 2 + nspec + 8
|
||||
bt = torch.arange(5 * width, dtype=torch.int32).reshape(5, width)
|
||||
sl = torch.tensor([0, 1, block_size, block_size * 3 + 1, 7], dtype=torch.int32)
|
||||
got = mamba_get_block_table_tensor(bt, sl, _spec(block_size, nspec), "align")
|
||||
want = _reference(bt, sl, block_size, nspec)
|
||||
assert torch.equal(got, want), f"cpu mismatch: {got.tolist()} != {want.tolist()}"
|
||||
assert got.device == bt.device
|
||||
assert got.dtype == bt.dtype
|
||||
|
||||
|
||||
def test_cpu_none_mode_passes_through():
|
||||
bt = torch.arange(12, dtype=torch.int32).reshape(3, 4)
|
||||
sl = torch.tensor([1, 2, 3], dtype=torch.int32)
|
||||
assert mamba_get_block_table_tensor(bt, sl, _spec(16, 3), "none") is bt
|
||||
@@ -171,6 +171,11 @@ def test_paged_prefill_indices_match_gathered_prefill(
|
||||
from vllm.v1.kv_cache_interface import SlidingWindowMLASpec
|
||||
|
||||
rocm = importlib.import_module(f"vllm.models.{model}.amd.rocm")
|
||||
builder_cls = (
|
||||
rocm.DeepseekV41ROCMAiterSparseSWAMetadataBuilder
|
||||
if model == "deepseek_v41"
|
||||
else rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder
|
||||
)
|
||||
device = torch.device("cuda")
|
||||
gen = torch.Generator().manual_seed(compress_ratio)
|
||||
window, width, block_size, max_model_len = 16, 32, 16, 128
|
||||
@@ -233,9 +238,7 @@ def test_paged_prefill_indices_match_gathered_prefill(
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_ops, "is_triton_sparse_mla_enabled", lambda: enabled
|
||||
)
|
||||
return rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder(
|
||||
swa_spec, ["swa"], vllm_config, device
|
||||
).build(0, common)
|
||||
return builder_cls(swa_spec, ["swa"], vllm_config, device).build(0, common)
|
||||
|
||||
assert build(enabled=False).prefill_swa_ragged_indices is None
|
||||
md = build(enabled=True)
|
||||
|
||||
@@ -8,6 +8,8 @@ from unittest.mock import MagicMock, patch
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.attention import AttentionConfig
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.selector import AttentionSelectorConfig
|
||||
@@ -44,6 +46,48 @@ def cleared_attention_selector_cache():
|
||||
_cached_get_attn_backend.cache_clear()
|
||||
|
||||
|
||||
def test_deepseek_rocm_backend_registry_paths_are_model_specific():
|
||||
assert AttentionBackendEnum.ROCM_FLASHMLA_SPARSE_DSV4.get_path() == (
|
||||
"vllm.models.deepseek_v4.amd.rocm.DeepseekV4ROCMAiterMLASparseBackend"
|
||||
)
|
||||
assert AttentionBackendEnum.ROCM_FLASHMLA_SPARSE_DSV41.get_path() == (
|
||||
"vllm.models.deepseek_v41.amd.rocm.DeepseekV41ROCMAiterMLASparseBackend"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend_name, expect_warning",
|
||||
[
|
||||
("ROCM_FLASHMLA_SPARSE_DSV4", True),
|
||||
("ROCM_FLASHMLA_SPARSE_DSV41", False),
|
||||
],
|
||||
ids=["legacy", "canonical"],
|
||||
)
|
||||
def test_deepseek_v41_rocm_accepts_legacy_backend_alias(backend_name, expect_warning):
|
||||
from vllm.models.deepseek_v41.amd import model as dsv41_model
|
||||
from vllm.models.deepseek_v41.amd.rocm import (
|
||||
DeepseekV41ROCMAiterMLAAttention,
|
||||
DeepseekV41ROCMAiterMLASparseBackend,
|
||||
)
|
||||
|
||||
config = SimpleNamespace(
|
||||
attention_config=AttentionConfig(backend=backend_name),
|
||||
parallel_config=ParallelConfig(),
|
||||
)
|
||||
|
||||
with patch.object(dsv41_model.logger, "warning_once") as warning_once:
|
||||
selected = dsv41_model._select_dsv4_attn_cls(config)
|
||||
|
||||
assert selected is DeepseekV41ROCMAiterMLAAttention
|
||||
if expect_warning:
|
||||
warning_once.assert_called_once()
|
||||
else:
|
||||
warning_once.assert_not_called()
|
||||
assert (
|
||||
DeepseekV41ROCMAiterMLASparseBackend.get_name() == "ROCM_FLASHMLA_SPARSE_DSV41"
|
||||
)
|
||||
|
||||
|
||||
def test_aiter_unified_attention_uses_dedicated_metadata_builder():
|
||||
from vllm.v1.attention.backends.rocm_aiter_unified_attn import (
|
||||
RocmAiterUnifiedAttentionBackend,
|
||||
|
||||
@@ -180,7 +180,7 @@ def create_standard_kv_cache_spec(
|
||||
|
||||
|
||||
def create_vllm_config(
|
||||
model_name: str = "meta-llama/Meta-Llama-3-8B",
|
||||
model_name: str = "Qwen/Qwen3-0.6B",
|
||||
tensor_parallel_size: int = 1,
|
||||
max_model_len: int = 1024,
|
||||
dtype: ModelDType | torch.dtype = "auto",
|
||||
|
||||
@@ -52,6 +52,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
hash_block_tokens,
|
||||
init_none_hash,
|
||||
make_block_hash_with_group_id,
|
||||
null_head_len,
|
||||
tensor_data,
|
||||
to_event_extra_keys,
|
||||
)
|
||||
@@ -2156,3 +2157,20 @@ def test_wrapped_mamba_group_requires_block_zeroing():
|
||||
)
|
||||
|
||||
assert config.needs_kv_cache_zeroing
|
||||
|
||||
|
||||
def test_null_head_len():
|
||||
null = KVCacheBlock(0, is_null=True)
|
||||
real = [KVCacheBlock(i) for i in range(1, 4)]
|
||||
tables = [
|
||||
[],
|
||||
real,
|
||||
[null] * 5,
|
||||
[null] * 5 + real,
|
||||
[null] * 2 + real[:1] + [null] * 3 + real[1:], # not one contiguous head
|
||||
]
|
||||
for table in tables:
|
||||
expected = next((i for i, b in enumerate(table) if not b.is_null), len(table))
|
||||
assert null_head_len(table) == expected
|
||||
blocks = KVCacheBlocks(([null] * 4 + real, [null] + real[:1] + [null]))
|
||||
assert blocks.get_unhashed_block_ids_all_groups() == [[1, 2, 3], [1]]
|
||||
|
||||
@@ -558,6 +558,249 @@ def test_schedule_prefills_gating(has_running: bool):
|
||||
assert any(r.req_id == "new0" for r in output.scheduled_new_reqs)
|
||||
|
||||
|
||||
def _decode_one(scheduler, output, req_ids: list[str]) -> None:
|
||||
"""Feed back one sampled token per request, so they keep decoding."""
|
||||
scheduler.update_from_output(
|
||||
output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=req_ids,
|
||||
req_id_to_index={req_id: i for i, req_id in enumerate(req_ids)},
|
||||
sampled_token_ids=[[0] for _ in req_ids],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _start_decoding(scheduler, req_id: str = "run0"):
|
||||
"""Prefill one request and take it into the decode state."""
|
||||
(request,) = create_requests(
|
||||
num_requests=1, num_tokens=8, max_tokens=100, req_ids=[req_id]
|
||||
)
|
||||
scheduler.add_request(request)
|
||||
output = scheduler.schedule()
|
||||
assert req_id in output.num_scheduled_tokens
|
||||
_decode_one(scheduler, output, [req_id])
|
||||
assert len(scheduler.running) == 1
|
||||
return output
|
||||
|
||||
|
||||
@pytest.mark.parametrize("interval", [2, 4])
|
||||
def test_local_prefill_interval_holds_prefill_between_bursts(interval: int):
|
||||
"""A single engine applies `prefill_schedule_interval` in the scheduler,
|
||||
timed from the last prefill-carrying step: a waiting prefill is held for the
|
||||
remainder of the interval while running requests keep decoding, then
|
||||
admitted once the interval has elapsed.
|
||||
"""
|
||||
scheduler = create_scheduler(prefill_schedule_interval=interval)
|
||||
# This prefill starts the interval.
|
||||
_start_decoding(scheduler)
|
||||
assert scheduler.last_prefill_step == scheduler.num_executed_steps
|
||||
|
||||
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
|
||||
scheduler.add_request(new_req)
|
||||
|
||||
# Steps inside the interval decode alone; the waiting prefill is held.
|
||||
for _ in range(interval - 1):
|
||||
output = scheduler.schedule()
|
||||
assert "new0" not in output.num_scheduled_tokens
|
||||
assert "run0" in output.num_scheduled_tokens
|
||||
assert new_req.status == RequestStatus.WAITING
|
||||
_decode_one(scheduler, output, ["run0"])
|
||||
|
||||
# The interval has now elapsed, so prefill is admitted again.
|
||||
output = scheduler.schedule()
|
||||
assert "new0" in output.num_scheduled_tokens
|
||||
assert scheduler.last_prefill_step == scheduler.num_executed_steps
|
||||
|
||||
|
||||
def test_empty_schedule_does_not_advance_the_prefill_cadence():
|
||||
"""An empty schedule() must not consume interval budget.
|
||||
|
||||
`current_step` counts every schedule() call. With async scheduling,
|
||||
step_with_batch_queue() calls schedule() again whenever the batch queue has
|
||||
room and has_requests() is true -- so a call lands while the previous batch
|
||||
is still in flight and nothing new can be scheduled. An async KV connector
|
||||
makes this common, since requests sit in lookup or WAITING_FOR_REMOTE_KVS.
|
||||
Timing the cadence on current_step lets those empty calls expire the hold
|
||||
early. Reported on #54627 against the LMCache MP connector.
|
||||
"""
|
||||
scheduler = create_scheduler(prefill_schedule_interval=4)
|
||||
_start_decoding(scheduler)
|
||||
|
||||
# One real decode step, left in flight (no update_from_output yet).
|
||||
output = scheduler.schedule()
|
||||
assert output.total_num_scheduled_tokens > 0
|
||||
executed_before = scheduler.num_executed_steps
|
||||
step_before = scheduler.current_step
|
||||
|
||||
# Re-entering schedule() while that batch is still unaccounted for
|
||||
# schedules nothing -- this is the async-scheduling shape.
|
||||
for _ in range(3):
|
||||
empty = scheduler.schedule()
|
||||
assert empty.total_num_scheduled_tokens == 0
|
||||
|
||||
assert scheduler.current_step == step_before + 3, "current_step should advance"
|
||||
assert scheduler.num_executed_steps == executed_before, (
|
||||
"an empty schedule() must not count as an executed step"
|
||||
)
|
||||
|
||||
|
||||
def test_prefill_cadence_counts_executed_steps_not_schedule_calls():
|
||||
"""The hold spans `interval` EXECUTED steps, however many empty schedule()
|
||||
calls are interleaved.
|
||||
|
||||
Counting schedule() calls instead lets each empty call eat interval budget,
|
||||
so with an async KV connector prefill lands on nearly every step and the
|
||||
feature stops doing anything.
|
||||
"""
|
||||
interval = 4
|
||||
scheduler = create_scheduler(prefill_schedule_interval=interval)
|
||||
_start_decoding(scheduler)
|
||||
prefill_at = scheduler.num_executed_steps
|
||||
|
||||
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
|
||||
scheduler.add_request(new_req)
|
||||
|
||||
empty_calls = 0
|
||||
admitted_at = None
|
||||
for _ in range(50):
|
||||
output = scheduler.schedule()
|
||||
if "new0" in output.num_scheduled_tokens:
|
||||
admitted_at = scheduler.num_executed_steps
|
||||
break
|
||||
if output.total_num_scheduled_tokens == 0:
|
||||
empty_calls += 1
|
||||
continue
|
||||
# A real decode step. Leave it in flight and re-enter schedule() twice,
|
||||
# as async scheduling does while the batch queue has room.
|
||||
for _ in range(2):
|
||||
extra = scheduler.schedule()
|
||||
if "new0" in extra.num_scheduled_tokens:
|
||||
admitted_at = scheduler.num_executed_steps
|
||||
break
|
||||
empty_calls += 1
|
||||
if admitted_at is not None:
|
||||
break
|
||||
_decode_one(scheduler, output, ["run0"])
|
||||
|
||||
assert admitted_at is not None, "prefill was never admitted"
|
||||
assert empty_calls > 0, "test never exercised an empty schedule() call"
|
||||
# The hold is measured in executed steps. Counting schedule() calls would
|
||||
# admit after far fewer, because the empty re-entries above would count.
|
||||
assert admitted_at - prefill_at == interval, (
|
||||
f"hold lasted {admitted_at - prefill_at} executed steps, expected "
|
||||
f"{interval}; empty schedule() calls were counted against the cadence"
|
||||
)
|
||||
assert scheduler.current_step > scheduler.num_executed_steps
|
||||
|
||||
|
||||
def test_local_prefill_interval_defaults_to_no_gating():
|
||||
"""The default interval of 1 leaves admission exactly as it was: a waiting
|
||||
prefill is admitted on the very next step.
|
||||
"""
|
||||
scheduler = create_scheduler()
|
||||
assert scheduler.local_prefill_interval == 1
|
||||
_start_decoding(scheduler)
|
||||
|
||||
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
|
||||
scheduler.add_request(new_req)
|
||||
output = scheduler.schedule()
|
||||
assert "new0" in output.num_scheduled_tokens
|
||||
|
||||
|
||||
def test_local_prefill_interval_left_to_engine_under_dp():
|
||||
"""Under data parallelism the cadence must stay aligned across ranks, so the
|
||||
DP engine core drives it and the scheduler must not gate on its own.
|
||||
"""
|
||||
scheduler = create_scheduler(prefill_schedule_interval=4, data_parallel_size=2)
|
||||
assert scheduler.local_prefill_interval == 1
|
||||
_start_decoding(scheduler)
|
||||
|
||||
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
|
||||
scheduler.add_request(new_req)
|
||||
output = scheduler.schedule()
|
||||
assert "new0" in output.num_scheduled_tokens
|
||||
|
||||
|
||||
def test_local_prefill_interval_never_idles_the_model():
|
||||
"""With nothing running to decode in prefill's place, the interval must not
|
||||
hold prefill back, or the step would be wasted entirely.
|
||||
"""
|
||||
scheduler = create_scheduler(prefill_schedule_interval=8)
|
||||
_start_decoding(scheduler)
|
||||
|
||||
# Finish the only running request, leaving nothing to decode.
|
||||
scheduler.finish_requests("run0", RequestStatus.FINISHED_ABORTED)
|
||||
assert not scheduler.running
|
||||
|
||||
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
|
||||
scheduler.add_request(new_req)
|
||||
output = scheduler.schedule()
|
||||
assert "new0" in output.num_scheduled_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize("interval", [2, 3])
|
||||
def test_local_prefill_interval_defers_inflight_prefill_chunk(interval: int):
|
||||
"""The single-engine interval gates prefill WORK, not just new admissions.
|
||||
|
||||
A request that is already partially prefilled is held for the remainder of
|
||||
the interval too, exactly as `throttle_prefills` does for DP. That is the
|
||||
point of the cadence -- a held step must carry no prefill at all, or decode
|
||||
does not get the uninterrupted stretch the interval exists to give it.
|
||||
"""
|
||||
scheduler = create_scheduler(
|
||||
max_num_seqs=16,
|
||||
max_num_batched_tokens=50,
|
||||
enable_chunked_prefill=True,
|
||||
prefill_schedule_interval=interval,
|
||||
)
|
||||
|
||||
# A short request that finishes prefill in one step -> a running decode.
|
||||
(decode_req,) = create_requests(num_requests=1, num_tokens=4, req_ids=["dec0"])
|
||||
scheduler.add_request(decode_req)
|
||||
output = scheduler.schedule()
|
||||
_decode_one(scheduler, output, ["dec0"])
|
||||
assert decode_req in scheduler.running and not decode_req.is_prefill_chunk
|
||||
|
||||
# A long request (80 tokens, budget 50) -> prefilled in chunks. That first
|
||||
# prefill closed the interval, so walk to the next step that opens it.
|
||||
(chunk_req,) = create_requests(num_requests=1, num_tokens=80, req_ids=["chk0"])
|
||||
scheduler.add_request(chunk_req)
|
||||
for _ in range(interval - 1):
|
||||
output = scheduler.schedule()
|
||||
assert "chk0" not in output.num_scheduled_tokens
|
||||
_decode_one(scheduler, output, ["dec0"])
|
||||
|
||||
output = scheduler.schedule()
|
||||
assert output.num_scheduled_tokens["chk0"] > 0
|
||||
scheduler.update_from_output(
|
||||
output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=["dec0", "chk0"],
|
||||
req_id_to_index={"dec0": 0, "chk0": 1},
|
||||
sampled_token_ids=[[0], []], # no token sampled for partial prefill
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
),
|
||||
)
|
||||
assert chunk_req.is_prefill_chunk # still mid-prefill, in running
|
||||
|
||||
# Held steps: the in-flight chunk is deferred along with new admissions,
|
||||
# while the decode keeps running.
|
||||
for _ in range(interval - 1):
|
||||
output = scheduler.schedule()
|
||||
assert "chk0" not in output.num_scheduled_tokens
|
||||
assert "dec0" in output.num_scheduled_tokens
|
||||
_decode_one(scheduler, output, ["dec0"])
|
||||
|
||||
# Once the interval has elapsed the chunk resumes.
|
||||
output = scheduler.schedule()
|
||||
assert "chk0" in output.num_scheduled_tokens
|
||||
|
||||
|
||||
def _setup_remote_kv_resume(num_prompt_tokens: int, matched_tokens: int):
|
||||
"""Drive a remote-KV request `r2` to the resume point (async load complete)
|
||||
while another request `r1` is already decoding, so the step is throttle-
|
||||
@@ -1916,6 +2159,46 @@ def test_draft_slots_budgeted_per_scheduled_request(tmp_path, monkeypatch):
|
||||
assert scheduler.schedule().num_scheduled_tokens == {"0": 10, "1": 4}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("method", "max_num_reqs"), [("dflash", 4), ("dspark", 5)])
|
||||
def test_block_drafters_cap_requests_instead_of_draft_slots(
|
||||
tmp_path, method, max_num_reqs
|
||||
):
|
||||
(tmp_path / "config.json").write_text(
|
||||
'{"architectures": ["OPTForCausalLM"], "model_type": "opt"}'
|
||||
)
|
||||
base = create_scheduler(
|
||||
model=str(tmp_path),
|
||||
max_num_batched_tokens=20,
|
||||
num_speculative_tokens=4,
|
||||
parallel_drafting=True,
|
||||
skip_tokenizer_init=True,
|
||||
)
|
||||
# DFlash/DSpark need a resolvable draft model, so switch the ngram config in
|
||||
# place and redo the config step. DSpark then has 4 lookahead tokens, not 5.
|
||||
vllm_config = base.vllm_config
|
||||
object.__setattr__(vllm_config.speculative_config, "method", method)
|
||||
object.__setattr__(
|
||||
vllm_config.speculative_config,
|
||||
"draft_model_config",
|
||||
SimpleNamespace(hf_config=SimpleNamespace()),
|
||||
)
|
||||
vllm_config._set_max_num_scheduled_tokens()
|
||||
scheduler = Scheduler(
|
||||
vllm_config=vllm_config,
|
||||
kv_cache_config=base.kv_cache_config,
|
||||
structured_output_manager=base.structured_output_manager,
|
||||
block_size=base.block_size,
|
||||
)
|
||||
for request in create_requests(num_requests=8, num_tokens=2):
|
||||
scheduler.add_request(request)
|
||||
|
||||
# Admission stops at 20 // num_lookahead_tokens, and no draft slots are
|
||||
# charged: each admitted prompt is scheduled in full.
|
||||
assert scheduler.schedule().num_scheduled_tokens == {
|
||||
str(i): 2 for i in range(max_num_reqs)
|
||||
}
|
||||
|
||||
|
||||
# Note - these test cases mirror some of those in test_rejection_sampler.py
|
||||
@pytest.mark.parametrize(
|
||||
"spec_tokens,output_tokens,expected,expected_per_req",
|
||||
|
||||
@@ -1013,6 +1013,7 @@ def test_predictor_matches_allocator_blocks_calculation_with_admission_cap():
|
||||
)
|
||||
total_computed = num_tokens
|
||||
|
||||
|
||||
def test_mamba_honors_drop_eagle_block():
|
||||
"""MambaManager must drop the EAGLE lookahead block, like FullAttentionManager.
|
||||
|
||||
@@ -1067,7 +1068,7 @@ def test_mamba_honors_drop_eagle_block():
|
||||
kv_cache_spec=spec,
|
||||
drop_eagle_block=drop_eagle_block,
|
||||
alignment_tokens=block_size,
|
||||
)[0]
|
||||
)[0][0]
|
||||
)
|
||||
|
||||
# Control: with no eagle drop, both groups see the whole cached prefix.
|
||||
@@ -1082,3 +1083,35 @@ def test_mamba_honors_drop_eagle_block():
|
||||
f"MambaManager ignored drop_eagle_block: full-attention dropped the "
|
||||
f"lookahead block (hit={full_hit}), mamba kept it (hit={mamba_hit})"
|
||||
)
|
||||
|
||||
|
||||
def test_pop_blocks_for_free_trims_null_head():
|
||||
block_size = 2
|
||||
spec = SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=4,
|
||||
)
|
||||
block_pool = BlockPool(
|
||||
num_gpu_blocks=10, enable_caching=True, hash_block_size=block_size
|
||||
)
|
||||
manager = get_sliding_window_manager(spec, block_pool)
|
||||
null = block_pool.null_block
|
||||
real = block_pool.get_new_blocks(2)
|
||||
manager.req_to_blocks["swa"] = [null] * 3 + real
|
||||
manager.req_to_blocks["dense"] = dense = block_pool.get_new_blocks(2)
|
||||
|
||||
assert manager.pop_blocks_for_free("swa") == real
|
||||
assert manager.pop_blocks_for_free("dense") is dense # no null head, no copy
|
||||
assert "swa" not in manager.req_to_blocks
|
||||
assert all(b.ref_cnt == 1 for b in real + dense)
|
||||
|
||||
# The null block is never pooled, so touch / free_blocks leave it alone.
|
||||
null_ref_cnt = null.ref_cnt
|
||||
num_free_blocks = block_pool.get_num_free_blocks()
|
||||
block_pool.touch([null])
|
||||
block_pool.free_blocks([null])
|
||||
assert null.ref_cnt == null_ref_cnt
|
||||
assert block_pool.get_num_free_blocks() == num_free_blocks
|
||||
|
||||
@@ -85,6 +85,7 @@ def create_scheduler(
|
||||
kv_cache_spec: KVCacheSpec | None = None,
|
||||
runner: RunnerOption = "auto",
|
||||
per_request_spec_decode_metrics: str = "none",
|
||||
prefill_schedule_interval: int = 1,
|
||||
scheduling_policy: SchedulerPolicy = "fcfs",
|
||||
diffusion_canvas_length: int | None = None,
|
||||
scheduler_cls: type[Scheduler] | None = None,
|
||||
@@ -134,6 +135,7 @@ def create_scheduler(
|
||||
enable_chunked_prefill=enable_chunked_prefill,
|
||||
async_scheduling=async_scheduling,
|
||||
is_encoder_decoder=model_config.is_encoder_decoder,
|
||||
prefill_schedule_interval=prefill_schedule_interval,
|
||||
# Ensure admission/preemption mechanics are deterministic
|
||||
watermark=0.0,
|
||||
policy=scheduling_policy,
|
||||
|
||||
@@ -160,6 +160,8 @@ def _make_manager_with_budgets(budgets: list[int]) -> EncoderCudaGraphManager:
|
||||
Skips the parts of __init__ that require a real VllmConfig / model
|
||||
by patching the attributes directly after construction.
|
||||
"""
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig
|
||||
|
||||
mgr = object.__new__(EncoderCudaGraphManager)
|
||||
mgr.token_budgets = sorted(budgets)
|
||||
mgr.path_token_budgets = {"default": mgr.token_budgets}
|
||||
@@ -1044,6 +1046,278 @@ class TestGetInputModality:
|
||||
assert model.get_input_modality(mm_kwargs) == "video"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GPU tests — DeepSeek-VL2 batched-tile encoder CUDA graph
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# DeepSeek-VL2 mock constants (kept small for fast capture)
|
||||
_DSV2_H = 2 # projector output side per tile (h = w)
|
||||
_DSV2_DIM = 16 # hidden size
|
||||
_DSV2_IMG_SIZE = 4 # each tile is 4×4 pixels
|
||||
_DSV2_BUDGETS = [16, 64]
|
||||
_DSV2_MAX_BATCH = 4
|
||||
|
||||
|
||||
def _dsv2_output_tokens(tw: int, th: int) -> int:
|
||||
"""Expected output tokens for an image with (tw, th) local tiles."""
|
||||
h = w = _DSV2_H
|
||||
return h * (w + 1) + th * h * (tw * w + 1) + 1
|
||||
|
||||
|
||||
def _make_deepseek_vl2_mm_kwargs(
|
||||
spatial_crops: list[list[int]],
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
) -> dict[str, Any]:
|
||||
"""spatial_crops: [[tw, th], ...] per image."""
|
||||
total_tiles = sum(1 + int(tw) * int(th) for tw, th in spatial_crops)
|
||||
pixel_values = torch.randn(
|
||||
total_tiles, 3, _DSV2_IMG_SIZE, _DSV2_IMG_SIZE, device=device, dtype=dtype
|
||||
)
|
||||
return {
|
||||
"pixel_values": pixel_values,
|
||||
"images_spatial_crop": torch.tensor(
|
||||
spatial_crops, dtype=torch.long, device=device
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class MockDeepseekVL2Model(torch.nn.Module, SupportsEncoderCudaGraph):
|
||||
"""Mock of DeepseekVLV2ForCausalLM for CUDA graph tests.
|
||||
|
||||
Simulates the batched-tile ViT pattern: pixel_values is
|
||||
[N_tiles, 3, H, W], output is assembled with newlines and
|
||||
view-separator in postprocess_encoder_output.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
hw = _DSV2_H * _DSV2_H
|
||||
in_features = 3 * _DSV2_IMG_SIZE * _DSV2_IMG_SIZE
|
||||
self.proj = torch.nn.Linear(in_features, hw * _DSV2_DIM)
|
||||
self.image_newline = torch.nn.Parameter(torch.randn(_DSV2_DIM))
|
||||
self.view_seperator = torch.nn.Parameter(torch.randn(_DSV2_DIM))
|
||||
self.global_view_pos = "head"
|
||||
|
||||
def get_max_frames_per_video(self) -> int:
|
||||
return 1
|
||||
|
||||
def get_encoder_cudagraph_config(self) -> EncoderCudaGraphConfig:
|
||||
return EncoderCudaGraphConfig(
|
||||
modalities=["image"],
|
||||
buffer_keys=["pixel_values"],
|
||||
out_hidden_size=_DSV2_DIM,
|
||||
)
|
||||
|
||||
def get_encoder_cudagraph_budget_range(self, vllm_config) -> tuple[int, int]:
|
||||
return (_dsv2_output_tokens(1, 1), 128)
|
||||
|
||||
def get_encoder_cudagraph_item_specs(
|
||||
self, mm_kwargs: dict[str, Any]
|
||||
) -> list[EncoderItemSpec]:
|
||||
h = w = _DSV2_H
|
||||
specs = []
|
||||
for row in mm_kwargs["images_spatial_crop"].tolist():
|
||||
tw, th = int(row[0]), int(row[1])
|
||||
if tw == 0 or th == 0:
|
||||
break
|
||||
specs.append(
|
||||
EncoderItemSpec(
|
||||
input_size=1 + tw * th,
|
||||
output_tokens=h * (w + 1) + th * h * (tw * w + 1) + 1,
|
||||
)
|
||||
)
|
||||
return specs
|
||||
|
||||
def select_encoder_cudagraph_items(
|
||||
self, mm_kwargs: dict[str, Any], indices: list[int]
|
||||
) -> dict[str, Any]:
|
||||
pixel_values = mm_kwargs["pixel_values"]
|
||||
images_spatial_crop = mm_kwargs["images_spatial_crop"]
|
||||
|
||||
if not indices:
|
||||
return {
|
||||
"pixel_values": pixel_values[:0],
|
||||
"images_spatial_crop": images_spatial_crop[:0],
|
||||
}
|
||||
|
||||
tiles_per_image = [
|
||||
1 + int(row[0]) * int(row[1]) for row in images_spatial_crop.tolist()
|
||||
]
|
||||
cum = [0]
|
||||
for t in tiles_per_image:
|
||||
cum.append(cum[-1] + t)
|
||||
|
||||
selected_pv = torch.cat([pixel_values[cum[i] : cum[i + 1]] for i in indices])
|
||||
return {
|
||||
"pixel_values": selected_pv,
|
||||
"images_spatial_crop": images_spatial_crop[list(indices)],
|
||||
}
|
||||
|
||||
def prepare_encoder_cudagraph_capture_inputs(
|
||||
self,
|
||||
token_budget: int,
|
||||
max_batch_size: int,
|
||||
max_frames_per_batch: int,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
path: str = "default",
|
||||
axis_keys: tuple[Hashable, ...] | None = None,
|
||||
) -> EncoderCudaGraphCaptureInputs:
|
||||
hw = _DSV2_H * _DSV2_H
|
||||
max_tiles = (token_budget + hw - 1) // hw
|
||||
dummy = torch.randn(
|
||||
max_tiles, 3, _DSV2_IMG_SIZE, _DSV2_IMG_SIZE, device=device, dtype=dtype
|
||||
)
|
||||
return EncoderCudaGraphCaptureInputs(values={"pixel_values": dummy})
|
||||
|
||||
def prepare_encoder_cudagraph_replay_buffers(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
max_batch_size: int,
|
||||
max_frames_per_batch: int,
|
||||
path: str = "default",
|
||||
) -> EncoderCudaGraphReplayBuffers:
|
||||
return EncoderCudaGraphReplayBuffers(
|
||||
values={"pixel_values": mm_kwargs["pixel_values"]}
|
||||
)
|
||||
|
||||
def _tile_forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
|
||||
"""Project tiles: [N, 3, H, W] → [N, hw, DIM]."""
|
||||
n = pixel_values.shape[0]
|
||||
flat = pixel_values.view(n, -1)
|
||||
hw = _DSV2_H * _DSV2_H
|
||||
return self.proj(flat).view(n, hw, _DSV2_DIM)
|
||||
|
||||
def encoder_cudagraph_forward(
|
||||
self, values: dict[str, torch.Tensor], path: str = "default"
|
||||
) -> torch.Tensor:
|
||||
return self._tile_forward(values["pixel_values"])
|
||||
|
||||
def encoder_eager_forward(
|
||||
self, mm_kwargs: dict[str, Any], path: str = "default"
|
||||
) -> torch.Tensor:
|
||||
# Manager always runs `postprocess_encoder_output` after this, whether
|
||||
# the batch went through eager or CUDA graph replay, so this must
|
||||
# return the same raw per-tile shape as `encoder_cudagraph_forward`.
|
||||
return self._tile_forward(mm_kwargs["pixel_values"])
|
||||
|
||||
def postprocess_encoder_output(
|
||||
self,
|
||||
outputs: dict[str, torch.Tensor],
|
||||
indices: list[int],
|
||||
per_item_out_tokens: list[int],
|
||||
dest: dict[int, torch.Tensor] | list[torch.Tensor | None],
|
||||
clone: bool = False,
|
||||
batch_mm_kwargs: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
assert batch_mm_kwargs is not None
|
||||
output = outputs["default"]
|
||||
images_spatial_crop = batch_mm_kwargs["images_spatial_crop"]
|
||||
_, hw, n_dim = output.shape
|
||||
h = w = int(hw**0.5)
|
||||
|
||||
tile_offset = 0
|
||||
for rank, img_idx in enumerate(indices):
|
||||
tw = int(images_spatial_crop[rank][0])
|
||||
th = int(images_spatial_crop[rank][1])
|
||||
n_tiles = 1 + tw * th
|
||||
tiles = output[tile_offset : tile_offset + n_tiles]
|
||||
tile_offset += n_tiles
|
||||
|
||||
global_f = tiles[0].view(h, w, n_dim)
|
||||
newline_g = self.image_newline.view(1, 1, n_dim).expand(h, 1, n_dim)
|
||||
global_f = torch.cat([global_f, newline_g], dim=1).view(-1, n_dim)
|
||||
|
||||
local_f = tiles[1:].view(th, tw, h, w, n_dim)
|
||||
local_f = local_f.permute(0, 2, 1, 3, 4).reshape(th * h, tw * w, n_dim)
|
||||
newline_l = self.image_newline.view(1, 1, n_dim).expand(th * h, 1, n_dim)
|
||||
local_f = torch.cat([local_f, newline_l], dim=1).view(-1, n_dim)
|
||||
|
||||
if self.global_view_pos == "head":
|
||||
emb = torch.cat([global_f, self.view_seperator[None], local_f])
|
||||
else:
|
||||
emb = torch.cat([local_f, self.view_seperator[None], global_f])
|
||||
|
||||
if isinstance(dest, dict):
|
||||
dest[img_idx] = emb.clone() if clone else emb
|
||||
else:
|
||||
dest[rank] = emb.clone() if clone else emb
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
|
||||
class TestDeepseekVL2CudaGraph:
|
||||
"""CUDA graph tests for the batched-tile encoder pattern (DeepSeek-VL2)."""
|
||||
|
||||
def setup_method(self):
|
||||
self.device = torch.device("cuda:0")
|
||||
self.dtype = torch.float16
|
||||
self.model = MockDeepseekVL2Model().to(self.device).half()
|
||||
self.mgr = _make_manager_for_gpu(
|
||||
self.model, _DSV2_BUDGETS, _DSV2_MAX_BATCH, self.device, self.dtype
|
||||
)
|
||||
self.graph_pool = current_platform.graph_pool_handle()
|
||||
self.mgr.capture(graph_pool=self.graph_pool)
|
||||
|
||||
def test_capture_creates_one_graph_per_budget(self):
|
||||
assert set(self.mgr.budget_graphs["default"].keys()) == set(_DSV2_BUDGETS)
|
||||
|
||||
def test_execute_returns_one_tensor_per_image(self):
|
||||
mm_kwargs = _make_deepseek_vl2_mm_kwargs(
|
||||
[[1, 1], [1, 1]], self.device, self.dtype
|
||||
)
|
||||
result = self.mgr.execute(mm_kwargs)
|
||||
assert result is not None
|
||||
assert len(result) == 2
|
||||
|
||||
def test_output_tokens_1x1_tile(self):
|
||||
# global=h*(w+1)=2*3=6, local=1*2*(1*2+1)=6, sep=1 → 13
|
||||
expected = _dsv2_output_tokens(1, 1)
|
||||
mm_kwargs = _make_deepseek_vl2_mm_kwargs([[1, 1]], self.device, self.dtype)
|
||||
result = self.mgr.execute(mm_kwargs)
|
||||
assert result is not None
|
||||
assert result[0].shape == (expected, _DSV2_DIM)
|
||||
|
||||
def test_output_tokens_2x1_tile(self):
|
||||
# global=6, local=1*2*(2*2+1)=10, sep=1 → 17
|
||||
expected = _dsv2_output_tokens(2, 1)
|
||||
mm_kwargs = _make_deepseek_vl2_mm_kwargs([[2, 1]], self.device, self.dtype)
|
||||
result = self.mgr.execute(mm_kwargs)
|
||||
assert result is not None
|
||||
assert result[0].shape == (expected, _DSV2_DIM)
|
||||
|
||||
def test_output_tokens_multi_image(self):
|
||||
# Image 0: tw=1,th=1 → 13; Image 1: tw=2,th=1 → 17
|
||||
e0 = _dsv2_output_tokens(1, 1)
|
||||
e1 = _dsv2_output_tokens(2, 1)
|
||||
mm_kwargs = _make_deepseek_vl2_mm_kwargs(
|
||||
[[1, 1], [2, 1]], self.device, self.dtype
|
||||
)
|
||||
result = self.mgr.execute(mm_kwargs)
|
||||
assert result is not None
|
||||
assert result[0].shape == (e0, _DSV2_DIM)
|
||||
assert result[1].shape == (e1, _DSV2_DIM)
|
||||
|
||||
def test_eager_fallback_when_tokens_exceed_all_budgets(self):
|
||||
# tw=4,th=4: global=6, local=4*2*(4*2+1)=72, sep=1 → 79 > max budget 64
|
||||
tw, th = 4, 4
|
||||
expected = _dsv2_output_tokens(tw, th)
|
||||
assert expected > _DSV2_BUDGETS[-1], "test precondition: must exceed max budget"
|
||||
mm_kwargs = _make_deepseek_vl2_mm_kwargs([[tw, th]], self.device, self.dtype)
|
||||
result = self.mgr.execute(mm_kwargs)
|
||||
assert result is not None
|
||||
assert len(result) == 1
|
||||
assert result[0].shape == (expected, _DSV2_DIM)
|
||||
assert self.mgr.graph_misses >= 1
|
||||
|
||||
def test_graph_hit_counter(self):
|
||||
mm_kwargs = _make_deepseek_vl2_mm_kwargs(
|
||||
[[1, 1], [1, 1]], self.device, self.dtype
|
||||
)
|
||||
self.mgr.execute(mm_kwargs)
|
||||
assert self.mgr.graph_hits == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GPU tests — video capture, replay, fallback, and mixed image+video
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -368,9 +368,11 @@ async def test_dp_pause_late_request_does_not_block_drain():
|
||||
# down first. The front-end sets its own copy optimistically when it
|
||||
# forwards the wake, so sample only after several coordinator
|
||||
# publishes (every 100ms while stats change) have overwritten it.
|
||||
# Rank 0 reports the wave only after its first step, which can outlast
|
||||
# the 2s on a cold device, so wait for the report rather than sample it.
|
||||
await asyncio.sleep(2)
|
||||
assert not long_request.done(), "the warmup request was too short to sample"
|
||||
assert engine.engine_core.dp_engines_running(), (
|
||||
assert await _poll_flag(engine, True, timeout=30), (
|
||||
"the coordinator does not report the engines as running while they are"
|
||||
)
|
||||
|
||||
|
||||
@@ -4,17 +4,28 @@ import asyncio
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from contextlib import AsyncExitStack
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import msgspec
|
||||
import openai # use the official client for correctness check
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.distributed.elastic_ep.external_elastic_ep import (
|
||||
ExternalElasticEPScaleCoordinator,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.engine.core_client import DPAsyncMPClient
|
||||
|
||||
MODEL_NAME = os.getenv("MODEL_NAME", "ibm-research/PowerMoE-3b")
|
||||
ELASTIC_EP_MODEL_NAME = "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
||||
|
||||
# Number of data parallel ranks for external LB testing
|
||||
DP_SIZE = int(os.getenv("DP_SIZE", "2"))
|
||||
@@ -22,6 +33,111 @@ DP_SIZE = int(os.getenv("DP_SIZE", "2"))
|
||||
TP_SIZE = int(os.getenv("TP_SIZE", "1"))
|
||||
|
||||
|
||||
class DictStore:
|
||||
def __init__(self, values: dict[str, bytes]):
|
||||
self.values = values
|
||||
|
||||
def check(self, keys: list[str]) -> bool:
|
||||
return all(key in self.values for key in keys)
|
||||
|
||||
def get(self, key: str) -> bytes:
|
||||
return self.values[key]
|
||||
|
||||
def set(self, key: str, value: bytes) -> None:
|
||||
self.values[key] = value
|
||||
|
||||
|
||||
def test_external_elastic_ep_visible_epoch_has_bootstrap_and_phase(monkeypatch):
|
||||
client = SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_master_port=12345,
|
||||
_data_parallel_master_port_list=[12346, 12347],
|
||||
)
|
||||
)
|
||||
)
|
||||
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
|
||||
control_store, reconfig_store = DictStore({}), DictStore({})
|
||||
coordinator.reconfig_store_ref = reconfig_store
|
||||
monkeypatch.setattr(
|
||||
coordinator, "_setup_reconfig_bootstrap", lambda: ("127.0.0.1", 12348)
|
||||
)
|
||||
epoch, bootstrap = coordinator._prepare_reconfig_bootstrap(control_store, 2, 3)
|
||||
|
||||
for store in (control_store, reconfig_store):
|
||||
# DictStore preserves insertion order: metadata must precede the epoch.
|
||||
assert list(store.values)[-1] == coordinator.key("current_epoch")
|
||||
assert store.get(coordinator.key(epoch, "bootstrap")) == msgspec.msgpack.encode(
|
||||
bootstrap
|
||||
)
|
||||
assert store.get(coordinator.key(epoch, "phase")) == b"preparing"
|
||||
assert store.get(coordinator.key("current_epoch")) == epoch.encode()
|
||||
monkeypatch.setattr(
|
||||
coordinator, "_get_reconfig_store", lambda store=store: store
|
||||
)
|
||||
assert coordinator.get_status()["requested_data_parallel_size"] == 3
|
||||
|
||||
|
||||
def test_external_elastic_ep_calculates_target_expert_redundancy():
|
||||
client = SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
model_config=SimpleNamespace(get_num_experts=lambda: 128),
|
||||
parallel_config=SimpleNamespace(
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=0)
|
||||
),
|
||||
)
|
||||
)
|
||||
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
|
||||
|
||||
assert coordinator._calculate_num_redundant_experts(2, 3) == 64
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("dp_rank", range(4))
|
||||
async def test_validate_expert_capacity_on_every_rank(dp_rank, monkeypatch):
|
||||
"""Validate expert capacity on every rank before waiting for an epoch."""
|
||||
client = SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
model_config=SimpleNamespace(get_num_experts=lambda: 128),
|
||||
parallel_config=SimpleNamespace(
|
||||
data_parallel_rank=dp_rank,
|
||||
elastic_ep_max_dp_size=4,
|
||||
data_parallel_master_ip="127.0.0.1",
|
||||
_coord_store_port=12345,
|
||||
eplb_config=SimpleNamespace(num_redundant_experts=0),
|
||||
),
|
||||
)
|
||||
)
|
||||
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
|
||||
store = DictStore({})
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.utils.get_cached_tcp_store_client", lambda *args: store
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="cannot retain all model experts"):
|
||||
await asyncio.wait_for(coordinator.prepare(4, 2), timeout=1)
|
||||
|
||||
assert not store.values
|
||||
assert coordinator.prepared_scale is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_external_elastic_ep_late_rank_observes_epoch_error():
|
||||
coordinator = ExternalElasticEPScaleCoordinator(
|
||||
cast("DPAsyncMPClient", SimpleNamespace())
|
||||
)
|
||||
epoch = "failed-epoch"
|
||||
store = DictStore(
|
||||
{
|
||||
coordinator.key("current_epoch"): epoch.encode(),
|
||||
coordinator.key(epoch, "error"): b"scale failed",
|
||||
}
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="scale failed"):
|
||||
await coordinator._wait_for_bootstrap(store, requested_new_dp_size=3)
|
||||
|
||||
|
||||
class ExternalLBServerManager:
|
||||
"""Manages data parallel vLLM server instances for external
|
||||
load balancer testing."""
|
||||
@@ -42,57 +158,69 @@ class ExternalLBServerManager:
|
||||
self.servers: list[tuple[RemoteOpenAIServer, list[str]]] = []
|
||||
self.server_threads: list[threading.Thread] = []
|
||||
|
||||
def start_rank(
|
||||
self,
|
||||
rank: int,
|
||||
dp_size: int | None = None,
|
||||
elastic_ep_scale_up_launch: bool = False,
|
||||
) -> tuple[RemoteOpenAIServer, list[str]]:
|
||||
"""Start one externally managed DP rank."""
|
||||
dp_size = self.dp_size if dp_size is None else dp_size
|
||||
server_args = self.base_server_args.copy()
|
||||
server_args.extend(
|
||||
[
|
||||
"--data-parallel-size",
|
||||
str(dp_size),
|
||||
"--data-parallel-rank",
|
||||
str(rank),
|
||||
"--data-parallel-size-local",
|
||||
"1",
|
||||
"--tensor-parallel-size",
|
||||
str(self.tp_size),
|
||||
"--port",
|
||||
str(8000 + rank),
|
||||
"--api-server-count",
|
||||
str(self.api_server_count),
|
||||
]
|
||||
)
|
||||
|
||||
env_dict = {
|
||||
"VLLM_SERVER_DEV_MODE": "1",
|
||||
current_platform.device_control_env_var: ",".join(
|
||||
str(current_platform.device_id_to_physical_device_id(i))
|
||||
for i in range(rank * self.tp_size, (rank + 1) * self.tp_size)
|
||||
),
|
||||
}
|
||||
if elastic_ep_scale_up_launch:
|
||||
env_dict["VLLM_ELASTIC_EP_SCALE_UP_LAUNCH"] = "1"
|
||||
|
||||
server = RemoteOpenAIServer(
|
||||
self.model_name,
|
||||
server_args,
|
||||
auto_port=False,
|
||||
env_dict=env_dict,
|
||||
)
|
||||
print(
|
||||
f"Server rank {rank} started successfully with "
|
||||
f"{self.api_server_count} API servers"
|
||||
)
|
||||
server_and_args = (server, server_args)
|
||||
self.servers.append(server_and_args)
|
||||
return server_and_args
|
||||
|
||||
def __enter__(self) -> list[tuple[RemoteOpenAIServer, list[str]]]:
|
||||
"""Start all server instances for external LB mode."""
|
||||
"""Start all initial server instances for external LB mode."""
|
||||
start_errors: list[Exception] = []
|
||||
for rank in range(self.dp_size):
|
||||
# Create server args for this specific rank
|
||||
server_args = self.base_server_args.copy()
|
||||
|
||||
# Add external LB specific arguments
|
||||
server_args.extend(
|
||||
[
|
||||
"--data-parallel-size",
|
||||
str(self.dp_size),
|
||||
"--data-parallel-rank",
|
||||
str(rank),
|
||||
"--data-parallel-size-local",
|
||||
"1",
|
||||
"--tensor-parallel-size",
|
||||
str(self.tp_size),
|
||||
"--port",
|
||||
str(8000 + rank), # Different port for each rank
|
||||
"--api-server-count",
|
||||
str(self.api_server_count),
|
||||
]
|
||||
)
|
||||
|
||||
# Use a thread to start each server to allow parallel initialization
|
||||
def start_server(r: int, sargs: list[str]):
|
||||
def start_server(r: int):
|
||||
try:
|
||||
# Start the server
|
||||
server = RemoteOpenAIServer(
|
||||
self.model_name,
|
||||
sargs,
|
||||
auto_port=False,
|
||||
env_dict={
|
||||
"VLLM_SERVER_DEV_MODE": "1",
|
||||
current_platform.device_control_env_var: ",".join(
|
||||
str(current_platform.device_id_to_physical_device_id(i))
|
||||
for i in range(r * TP_SIZE, (r + 1) * TP_SIZE)
|
||||
),
|
||||
},
|
||||
)
|
||||
server.__enter__()
|
||||
print(
|
||||
f"Server rank {r} started successfully with "
|
||||
f"{self.api_server_count} API servers"
|
||||
)
|
||||
self.servers.append((server, sargs))
|
||||
self.start_rank(r)
|
||||
except Exception as e:
|
||||
print(f"Failed to start server rank {r}: {e}")
|
||||
raise
|
||||
start_errors.append(e)
|
||||
|
||||
thread = threading.Thread(target=start_server, args=(rank, server_args))
|
||||
thread = threading.Thread(target=start_server, args=(rank,))
|
||||
thread.start()
|
||||
|
||||
self.server_threads.append(thread)
|
||||
@@ -101,6 +229,10 @@ class ExternalLBServerManager:
|
||||
for thread in self.server_threads:
|
||||
thread.join()
|
||||
|
||||
if start_errors:
|
||||
start_error = start_errors[0]
|
||||
raise RuntimeError("Failed to start external LB servers") from start_error
|
||||
|
||||
# Give servers additional time to fully initialize and coordinate
|
||||
time.sleep(2)
|
||||
|
||||
@@ -184,12 +316,104 @@ def _assert_expert_parallel_config(
|
||||
assert parallel_config["enable_expert_parallel"] is expected
|
||||
|
||||
|
||||
def _send_scale_command(
|
||||
server: RemoteOpenAIServer, new_dp_size: int
|
||||
) -> requests.Response:
|
||||
return requests.post(
|
||||
server.url_for("scale_elastic_ep"),
|
||||
json={"new_data_parallel_size": new_dp_size},
|
||||
timeout=600,
|
||||
)
|
||||
|
||||
|
||||
def _wait_for_scale_requests_to_start(
|
||||
scale_requests: list[Future[requests.Response]],
|
||||
timeout: float = 30,
|
||||
) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
for request in scale_requests:
|
||||
if request.done():
|
||||
response = request.result()
|
||||
pytest.fail(
|
||||
"Scale request finished before the new rank was launched: "
|
||||
f"{response.status_code} {response.text}"
|
||||
)
|
||||
|
||||
if all(request.running() for request in scale_requests):
|
||||
return
|
||||
time.sleep(0.1)
|
||||
|
||||
pytest.fail("Timed out waiting for all scale requests to start")
|
||||
|
||||
|
||||
@pytest.mark.distributed(num_gpus=3)
|
||||
@pytest.mark.skipif(
|
||||
TP_SIZE != 1 or current_platform.device_count() < 3,
|
||||
reason="External Elastic EP scale-up requires three GPUs with TP_SIZE=1",
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
os.getenv("VLLM_USE_RUST_FRONTEND") == "1",
|
||||
reason="Elastic EP scaling is not supported by the Rust frontend",
|
||||
)
|
||||
def test_external_lb_elastic_ep_scale_up(default_server_args) -> None:
|
||||
server_args = [
|
||||
*default_server_args,
|
||||
"--enable-expert-parallel",
|
||||
"--all2all-backend",
|
||||
"allgather_reducescatter",
|
||||
"--attention-backend",
|
||||
"TRITON_MLA",
|
||||
"--enable-elastic-ep",
|
||||
"--elastic-ep-max-dp-size",
|
||||
"3",
|
||||
"--enable-eplb",
|
||||
"--eplb-config",
|
||||
'{"communicator":"pynccl","num_redundant_experts":0,"use_async":false}',
|
||||
]
|
||||
server_manager = ExternalLBServerManager(
|
||||
ELASTIC_EP_MODEL_NAME,
|
||||
dp_size=2,
|
||||
api_server_count=1,
|
||||
base_server_args=server_args,
|
||||
)
|
||||
|
||||
with server_manager:
|
||||
existing_servers = [server for server, _ in server_manager.servers]
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
scale_requests = [
|
||||
executor.submit(_send_scale_command, server, 3)
|
||||
for server in existing_servers
|
||||
]
|
||||
_wait_for_scale_requests_to_start(scale_requests)
|
||||
server_manager.start_rank(
|
||||
rank=2,
|
||||
dp_size=3,
|
||||
elastic_ep_scale_up_launch=True,
|
||||
)
|
||||
|
||||
for request in scale_requests:
|
||||
response = request.result(timeout=600)
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
for server, _ in server_manager.servers:
|
||||
parallel_config = _get_parallel_config(server)
|
||||
assert parallel_config["data_parallel_size"] == 3
|
||||
assert parallel_config["eplb_config"]["num_redundant_experts"] == 32
|
||||
with server.get_client() as client:
|
||||
completion = client.completions.create(
|
||||
model=ELASTIC_EP_MODEL_NAME,
|
||||
prompt="Hello, my name is",
|
||||
max_tokens=5,
|
||||
temperature=0.0,
|
||||
)
|
||||
assert completion.choices[0].finish_reason in ("length", "stop")
|
||||
|
||||
|
||||
def test_external_lb_server_info(server_manager):
|
||||
servers = server_manager.servers
|
||||
api_server_count = server_manager.api_server_count
|
||||
|
||||
_assert_expert_parallel_config(servers)
|
||||
|
||||
for i, (server, _) in enumerate(servers):
|
||||
print(f"Testing {i=}")
|
||||
|
||||
|
||||
@@ -5,7 +5,13 @@ import pytest
|
||||
|
||||
from tests.utils import single_gpu_only
|
||||
from vllm import SamplingParams
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...utils import (
|
||||
assert_request_outputs_match,
|
||||
compute_acceptance_len,
|
||||
get_test_prompts,
|
||||
)
|
||||
from .._correctness import check_mtp_correctness
|
||||
|
||||
|
||||
@@ -37,3 +43,107 @@ def test_mtp_correctness(
|
||||
expected_accuracy_threshold,
|
||||
vllm_runner,
|
||||
)
|
||||
|
||||
|
||||
@single_gpu_only
|
||||
def test_mtp_draft_lm_head_quantization(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sampling_config: SamplingParams,
|
||||
vllm_runner,
|
||||
):
|
||||
"""A quantized draft copy of the shared lm_head only changes drafting:
|
||||
greedy outputs match stock MTP and acceptance stays within 10%."""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
prompts = get_test_prompts(mm_enabled=False)
|
||||
results = {}
|
||||
# NVFP4 needs a CUDA W4A16 kernel.
|
||||
quants = (None, "fp8", "nvfp4") if current_platform.is_cuda() else (None, "fp8")
|
||||
for quant in quants:
|
||||
with vllm_runner(
|
||||
"Qwen/Qwen3.5-0.8B-Base",
|
||||
block_size=None,
|
||||
enable_chunked_prefill=None,
|
||||
max_model_len=2048,
|
||||
limit_mm_per_prompt={"image": 0, "video": 0},
|
||||
speculative_config={
|
||||
"method": "mtp",
|
||||
"num_speculative_tokens": 3,
|
||||
"draft_lm_head_quantization": quant,
|
||||
},
|
||||
disable_log_stats=False,
|
||||
) as runner:
|
||||
outputs = runner.llm.chat(prompts, sampling_config)
|
||||
results[quant] = (outputs, compute_acceptance_len(runner.llm.get_metrics()))
|
||||
|
||||
ref_outputs, ref_al = results.pop(None)
|
||||
for quant, (outputs, al) in results.items():
|
||||
assert_request_outputs_match(
|
||||
ref_outputs,
|
||||
outputs,
|
||||
# Not bit-exact: a different acceptance length changes the target's
|
||||
# verify batch shapes, which can flip a near-tied greedy token.
|
||||
required_matches=int(0.8 * len(ref_outputs)) + 1,
|
||||
context=f"draft_lm_head_quantization={quant}",
|
||||
)
|
||||
assert al >= 0.9 * ref_al, (
|
||||
f"{quant}: acceptance length {al:.3f} vs {ref_al:.3f}"
|
||||
)
|
||||
|
||||
|
||||
@single_gpu_only
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="nvfp4 needs a CUDA W4A16 kernel"
|
||||
)
|
||||
def test_mtp_draft_lm_head_quantization_survives_sleep_and_reload(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
sampling_config: SamplingParams,
|
||||
vllm_runner,
|
||||
):
|
||||
"""The quantized draft head is derived from the target head: it must come
|
||||
back intact after a level 1 sleep and be re-derived after a target weight
|
||||
reload, so outputs and acceptance match the run before."""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
prompts = get_test_prompts(mm_enabled=False)
|
||||
with vllm_runner(
|
||||
"Qwen/Qwen3.5-0.8B-Base",
|
||||
block_size=None,
|
||||
enable_chunked_prefill=None,
|
||||
max_model_len=2048,
|
||||
limit_mm_per_prompt={"image": 0, "video": 0},
|
||||
speculative_config={
|
||||
"method": "mtp",
|
||||
"num_speculative_tokens": 3,
|
||||
"draft_lm_head_quantization": "nvfp4",
|
||||
},
|
||||
enable_sleep_mode=True,
|
||||
disable_log_stats=False,
|
||||
) as runner:
|
||||
llm = runner.llm
|
||||
|
||||
def run():
|
||||
before = llm.get_metrics()
|
||||
outputs = llm.chat(prompts, sampling_config)
|
||||
return outputs, compute_acceptance_len(llm.get_metrics(), before)
|
||||
|
||||
ref_outputs, ref_al = run()
|
||||
|
||||
llm.sleep(level=1)
|
||||
llm.wake_up()
|
||||
outputs, al = run()
|
||||
assert_request_outputs_match(
|
||||
ref_outputs,
|
||||
outputs,
|
||||
required_matches=int(0.8 * len(ref_outputs)) + 1,
|
||||
context="after level 1 sleep",
|
||||
)
|
||||
assert al == pytest.approx(ref_al, rel=0.03)
|
||||
|
||||
llm.collective_rpc("reload_weights")
|
||||
outputs, al = run()
|
||||
assert_request_outputs_match(
|
||||
ref_outputs,
|
||||
outputs,
|
||||
required_matches=int(0.8 * len(ref_outputs)) + 1,
|
||||
context="after weight reload",
|
||||
)
|
||||
assert al == pytest.approx(ref_al, rel=0.03)
|
||||
|
||||
@@ -141,7 +141,7 @@ def _make_addresses() -> EngineZmqAddresses:
|
||||
False,
|
||||
),
|
||||
({"data_parallel_backend": "mp", "data_parallel_rank_local": 0}, False),
|
||||
({"data_parallel_backend": "mp", "enable_elastic_ep": True}, False),
|
||||
({"data_parallel_backend": "mp", "enable_elastic_ep": True}, True),
|
||||
({"data_parallel_backend": "mp", "data_parallel_size": 1}, False),
|
||||
],
|
||||
ids=["mp", "ray", "non-master-node", "offline", "elastic-ep", "single-engine"],
|
||||
@@ -150,8 +150,9 @@ def test_coordination_store_held_only_by_the_online_dp_master(
|
||||
monkeypatch: pytest.MonkeyPatch, overrides: dict[str, Any], expect_store: bool
|
||||
) -> None:
|
||||
"""The DP master node holds a coordination store for both engine backends,
|
||||
so engines there pick world-group ports at bind time. Every other
|
||||
deployment keeps the pre-allocated ports and must not hold one."""
|
||||
so engines there pick world-group ports at bind time. Elastic EP also
|
||||
creates its coordination store before startup. The remaining cases
|
||||
below keep the pre-allocated ports and must not hold a store."""
|
||||
|
||||
class FakeManager:
|
||||
def __init__(self, **kwargs) -> None:
|
||||
|
||||
@@ -142,6 +142,9 @@ def test_mp_client_uses_env_timeout(monkeypatch: pytest.MonkeyPatch):
|
||||
def setsockopt(self, *_args, **_kwargs):
|
||||
pass
|
||||
|
||||
def getsockopt_string(self, _option):
|
||||
return "inproc://dummy"
|
||||
|
||||
monkeypatch.setattr(core_client_mod.zmq.Socket, "shadow", lambda *_: ShadowSocket())
|
||||
monkeypatch.setattr(
|
||||
core_client_mod, "make_zmq_socket", lambda *_, **__: DummySocket()
|
||||
@@ -210,6 +213,7 @@ def _make_dplb_client(num_engines: int = 3, client_count: int = 1) -> DPLBAsyncM
|
||||
client = object.__new__(DPLBAsyncMPClient)
|
||||
client.client_count = client_count
|
||||
client.reqs_in_flight = {}
|
||||
client.streaming_reqs = set()
|
||||
client.engine_inflight = Counter()
|
||||
client.core_engines = [bytes([i, 0]) for i in range(num_engines)]
|
||||
client.lb_engines = [[0, 0, 0.0] for _ in range(num_engines)]
|
||||
@@ -407,11 +411,47 @@ def test_dplb_finished_requests_release_inflight():
|
||||
assert req.request_id not in client.reqs_in_flight
|
||||
|
||||
|
||||
@pytest.mark.parametrize("final_sent", [False, True])
|
||||
def test_dplb_streaming_session_abort_goes_to_all_engines(final_sent: bool):
|
||||
"""A streaming-input chunk in flight when its session finished may
|
||||
re-create it on any engine, so an open session's abort reaches every
|
||||
engine. Other aborts go only to the routed engine."""
|
||||
client = _make_dplb_client(num_engines=3)
|
||||
client.resources = SimpleNamespace(engine_dead=False)
|
||||
sent: dict[bytes, list[str]] = {}
|
||||
|
||||
async def abort_requests(request_ids, engine):
|
||||
sent[engine] = request_ids
|
||||
|
||||
client._abort_requests = abort_requests
|
||||
|
||||
chunk = make_request(SamplingParams(max_tokens=1))
|
||||
chunk.resumable = True
|
||||
client.get_core_engine_for_request(chunk)
|
||||
if final_sent:
|
||||
chunk.resumable = False
|
||||
client.get_core_engine_for_request(chunk)
|
||||
# The engine finished the session, dropping its routing entry.
|
||||
outputs = EngineCoreOutputs(finished_requests={chunk.request_id})
|
||||
asyncio.run(DPLBAsyncMPClient.process_engine_outputs(client, outputs))
|
||||
client.reqs_in_flight["live"] = client.core_engines[1]
|
||||
|
||||
asyncio.run(client.abort_requests_async([chunk.request_id, "live"]))
|
||||
|
||||
if final_sent:
|
||||
expected = {client.core_engines[1]: ["live"]}
|
||||
else:
|
||||
expected = {engine: [chunk.request_id] for engine in client.core_engines}
|
||||
expected[client.core_engines[1]].append("live")
|
||||
assert sent == expected
|
||||
assert not client.streaming_reqs
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("effective_size", "other_size"),
|
||||
[(None, None), (4224, 4224), (4224, 1056), (4224, None)],
|
||||
)
|
||||
def test_apply_ready_response_syncs_block_size(effective_size, other_size):
|
||||
def test_apply_ready_response_syncs_engine_metadata(effective_size, other_size):
|
||||
import msgspec
|
||||
|
||||
client = object.__new__(MPClient)
|
||||
@@ -419,8 +459,11 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
parallel_config=SimpleNamespace(_coord_store_port=0),
|
||||
)
|
||||
client.stats_update_address = None
|
||||
client.coordinator_input_address = None
|
||||
client.coordinator_output_address = None
|
||||
|
||||
payload = msgspec.msgpack.encode(
|
||||
EngineCoreReadyResponse(
|
||||
@@ -441,6 +484,9 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
|
||||
instance_id="test-instance",
|
||||
supports_lora=False,
|
||||
max_loras=0,
|
||||
coord_store_port=1234,
|
||||
coordinator_input_address="tcp://127.0.0.1:1235",
|
||||
coordinator_output_address="tcp://127.0.0.1:1236",
|
||||
)
|
||||
)
|
||||
fields = msgspec.msgpack.decode(payload)
|
||||
@@ -461,6 +507,9 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
|
||||
|
||||
client._apply_ready_response(b"")
|
||||
assert cache_config.effective_attention_block_size is None
|
||||
assert client.vllm_config.parallel_config._coord_store_port == 1234
|
||||
assert client.coordinator_input_address == "tcp://127.0.0.1:1235"
|
||||
assert client.coordinator_output_address == "tcp://127.0.0.1:1236"
|
||||
|
||||
|
||||
def test_apply_ready_response_syncs_mamba_block_size():
|
||||
@@ -471,6 +520,7 @@ def test_apply_ready_response_syncs_mamba_block_size():
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
parallel_config=SimpleNamespace(_coord_store_port=0),
|
||||
)
|
||||
client.stats_update_address = None
|
||||
|
||||
@@ -1541,6 +1591,7 @@ def test_apply_ready_response_retains_kv_event_sources():
|
||||
client.vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
parallel_config=SimpleNamespace(_coord_store_port=0),
|
||||
)
|
||||
|
||||
def ready(rank: int, config: KVEventsConfig | None) -> bytes:
|
||||
|
||||
@@ -42,6 +42,7 @@ from vllm.v1.engine.output_processor import (
|
||||
OutputProcessor,
|
||||
RequestOutputCollector,
|
||||
RequestState,
|
||||
StreamingInputOutputCollector,
|
||||
)
|
||||
from vllm.v1.engine.parallel_sampling import ParentRequest
|
||||
from vllm.v1.metrics.loggers import PrometheusStatLogger
|
||||
@@ -1642,6 +1643,137 @@ async def test_cumulative_output_collector_n():
|
||||
assert third[0].text == "c"
|
||||
|
||||
|
||||
def _make_boundary_output(
|
||||
token_id: int, finish_reason: str | None, finished: bool = False
|
||||
) -> RequestOutput:
|
||||
return RequestOutput(
|
||||
request_id="my-request-id",
|
||||
prompt=None,
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text="a",
|
||||
token_ids=[token_id],
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
],
|
||||
finished=finished,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_input_collector_preserves_chunk_boundary():
|
||||
"""A pending chunk-boundary marker (finish_reason on a non-finished
|
||||
output) must not be erased by merging the next chunk's output into it."""
|
||||
collector = StreamingInputOutputCollector(
|
||||
RequestOutputKind.DELTA, request_id="my-request-id-int"
|
||||
)
|
||||
|
||||
collector.put(_make_boundary_output(0, "stop"))
|
||||
collector.put(_make_boundary_output(1, None))
|
||||
collector.put(_make_boundary_output(2, None))
|
||||
|
||||
boundary = await collector.get()
|
||||
assert boundary.outputs[0].finish_reason == "stop"
|
||||
assert boundary.outputs[0].token_ids == [0]
|
||||
|
||||
following = await collector.get()
|
||||
assert following.outputs[0].finish_reason is None
|
||||
assert following.outputs[0].token_ids == [1, 2]
|
||||
|
||||
assert collector.output is None
|
||||
assert not collector._backlog
|
||||
assert not collector.ready.is_set()
|
||||
|
||||
|
||||
def _make_streaming_input_request(
|
||||
request_id: str, prompt_token_ids: list[int]
|
||||
) -> EngineCoreRequest:
|
||||
"""One chunk of a streaming-input (resumable) request."""
|
||||
return EngineCoreRequest(
|
||||
request_id=request_id,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
mm_features=None,
|
||||
sampling_params=SamplingParams(
|
||||
detokenize=False, output_kind=RequestOutputKind.DELTA
|
||||
),
|
||||
pooling_params=None,
|
||||
arrival_time=0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
resumable=True,
|
||||
external_req_id=f"{request_id}-ext",
|
||||
)
|
||||
|
||||
|
||||
def _add_streaming_request_with_queued_chunk(
|
||||
processor: OutputProcessor, request_id: str
|
||||
) -> RequestOutputCollector:
|
||||
queue = RequestOutputCollector(RequestOutputKind.DELTA, request_id=request_id)
|
||||
processor.add_request(
|
||||
_make_streaming_input_request(request_id, [1, 2, 3]), None, queue=queue
|
||||
)
|
||||
processor.add_request(_make_streaming_input_request(request_id, [7, 8]), None)
|
||||
return queue
|
||||
|
||||
|
||||
def test_streaming_input_stop_is_chunk_boundary():
|
||||
"""STOP is a per-chunk boundary: finished is forced False, the queued
|
||||
chunk is applied, and the request state stays alive."""
|
||||
processor = OutputProcessor(None, log_stats=False)
|
||||
queue = _add_streaming_request_with_queued_chunk(processor, "stream-req")
|
||||
|
||||
processed = processor.process_outputs(
|
||||
[
|
||||
EngineCoreOutput(
|
||||
request_id="stream-req",
|
||||
new_token_ids=[4],
|
||||
finish_reason=FinishReason.STOP,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
assert processed.reqs_to_abort == []
|
||||
assert processor.get_num_unfinished_requests() == 1
|
||||
output = queue.get_nowait()
|
||||
assert output is not None
|
||||
assert output.finished is False
|
||||
assert output.outputs[0].finish_reason == "stop"
|
||||
req_state = processor.request_states["stream-req"]
|
||||
assert req_state.prompt_token_ids is not None
|
||||
assert req_state.prompt_token_ids[-2:] == [7, 8]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("finish_reason", [FinishReason.ABORT, FinishReason.ERROR])
|
||||
def test_streaming_input_abort_error_is_terminal(finish_reason: FinishReason):
|
||||
"""ABORT/ERROR is terminal, not a chunk boundary: finished stays True,
|
||||
the state is freed, and an abort is routed back to the engine core."""
|
||||
processor = OutputProcessor(None, log_stats=False)
|
||||
queue = _add_streaming_request_with_queued_chunk(processor, "stream-req")
|
||||
|
||||
processed = processor.process_outputs(
|
||||
[
|
||||
EngineCoreOutput(
|
||||
request_id="stream-req",
|
||||
new_token_ids=[],
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
assert processor.get_num_unfinished_requests() == 0
|
||||
assert processed.reqs_to_abort == ["stream-req"]
|
||||
output = queue.get_nowait()
|
||||
assert output is not None
|
||||
assert output.finished is True
|
||||
assert output.outputs[0].finish_reason == str(finish_reason)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner", ["generate", "pooling"])
|
||||
@pytest.mark.parametrize("abort_by", ["internal", "external"])
|
||||
def test_abort_requests(runner: str, abort_by: str, dummy_test_vectors):
|
||||
|
||||
@@ -281,6 +281,7 @@ class _FinishedProcess:
|
||||
|
||||
|
||||
def test_wait_for_engine_startup_reports_watched_process_exit():
|
||||
"""Update the startup test argument so it reaches frontend failure handling."""
|
||||
ctx = zmq.Context()
|
||||
handshake_socket = ctx.socket(zmq.ROUTER)
|
||||
recv, send = connection.Pipe(duplex=False)
|
||||
|
||||
@@ -190,11 +190,13 @@ def _apply_ft(server, instruction: str, params: dict | None = None) -> dict:
|
||||
|
||||
def _kill_worker_process(server) -> None:
|
||||
"""SIGKILL only the worker proc, leaving EngineCore and API server alive."""
|
||||
workers = [
|
||||
p
|
||||
for p in psutil.Process(server.proc.pid).children(recursive=True)
|
||||
if "Worker" in " ".join(p.cmdline())
|
||||
]
|
||||
workers = []
|
||||
for p in psutil.Process(server.proc.pid).children(recursive=True):
|
||||
# Skip descendants that already exited: cmdline() raises ZombieProcess
|
||||
# if unreaped, NoSuchProcess if reaped. A dead worker still fails below.
|
||||
with contextlib.suppress(psutil.ZombieProcess, psutil.NoSuchProcess):
|
||||
if "Worker" in " ".join(p.cmdline()):
|
||||
workers.append(p)
|
||||
assert len(workers) == 1, f"expected 1 worker proc, found: {workers}"
|
||||
workers[0].kill()
|
||||
|
||||
|
||||
@@ -73,6 +73,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
KVQuantMode,
|
||||
MLAAttentionSpec,
|
||||
compute_layer_kv_cache_shape_bytes,
|
||||
)
|
||||
@@ -1551,6 +1552,95 @@ class TestNixlHandshake:
|
||||
with pytest.raises(AssertionError):
|
||||
worker.add_remote_agent(bad_meta, remote_tp_size=2)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kv_quant_mode", "local_tp", "remote_tp", "num_kv_heads", "rejected"),
|
||||
[
|
||||
(KVQuantMode.NVFP4, 2, 1, 8, True),
|
||||
(KVQuantMode.NVFP4, 1, 2, 8, True),
|
||||
(KVQuantMode.NVFP4, 2, 2, 8, False),
|
||||
# Both sides hold one replicated head per rank: no head split.
|
||||
(KVQuantMode.NVFP4, 16, 8, 8, False),
|
||||
(KVQuantMode.NONE, 2, 1, 8, False),
|
||||
],
|
||||
)
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FakeNixlWrapper,
|
||||
)
|
||||
def test_handshake_rejects_nvfp4_kv_head_split(
|
||||
self,
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
kv_quant_mode,
|
||||
local_tp,
|
||||
remote_tp,
|
||||
num_kv_heads,
|
||||
rejected,
|
||||
):
|
||||
"""NVFP4 pages keep all heads' data before all heads' scales, so
|
||||
heterogeneous-TP head splitting would silently corrupt the KV.
|
||||
"""
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(block_size=block_size)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=1,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=max(1, num_kv_heads // local_tp),
|
||||
head_size=16,
|
||||
dtype=torch.uint8,
|
||||
kv_quant_mode=kv_quant_mode,
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
with patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501
|
||||
return_value=local_tp,
|
||||
):
|
||||
worker = FakeNixlConnectorWorker(
|
||||
vllm_config,
|
||||
"engine",
|
||||
hand_shake_latency=0,
|
||||
kv_cache_config=kv_cache_config,
|
||||
)
|
||||
worker.transfer_topo.total_num_kv_heads = num_kv_heads
|
||||
worker.transfer_topo.local_physical_heads = max(1, num_kv_heads // local_tp)
|
||||
|
||||
local_len = 4096 * worker.block_size
|
||||
remote_len = (
|
||||
local_len
|
||||
* max(1, num_kv_heads // remote_tp)
|
||||
// worker.transfer_topo.local_physical_heads
|
||||
)
|
||||
worker.block_len_per_layer = [local_len]
|
||||
worker.num_blocks = 1
|
||||
worker.dst_num_blocks[worker.engine_id] = worker.num_blocks
|
||||
meta = NixlAgentMetadata(
|
||||
engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
|
||||
agent_metadata=FakeNixlWrapper.AGENT_METADATA,
|
||||
kv_caches_base_addr=[0],
|
||||
device_id=0,
|
||||
num_blocks=1,
|
||||
block_lens=[remote_len],
|
||||
block_strides=[remote_len],
|
||||
kv_cache_layout=worker.kv_cache_layout,
|
||||
block_size=worker.block_size,
|
||||
ssm_sizes=(0, 0),
|
||||
attn_backend_name=worker.backend_name,
|
||||
physical_blocks_per_logical_kv_block=1,
|
||||
)
|
||||
|
||||
if rejected:
|
||||
with pytest.raises(RuntimeError, match="NVFP4"):
|
||||
worker.add_remote_agent(meta, remote_tp_size=remote_tp)
|
||||
else:
|
||||
worker.add_remote_agent(meta, remote_tp_size=remote_tp)
|
||||
|
||||
|
||||
# NOTE: resource cleanup in mp backend is a bit finicky, so the order in which
|
||||
# we put here is important. First run ray, it will clean up the resources, then
|
||||
|
||||
@@ -653,6 +653,41 @@ def test_madvise_unexpected_oserror_propagates(iid, monkeypatch):
|
||||
assert exc_info.value.errno == errno.EIO
|
||||
|
||||
|
||||
def test_segmented_region_maps_files_back_to_back(iid, monkeypatch):
|
||||
"""Rows split over three shm files mapped back to back: each worker sees
|
||||
the other's writes in every file, and the creator removes every file."""
|
||||
monkeypatch.setattr(region_module, "_MIN_SEGMENT_BYTES", PAGE_SIZE)
|
||||
regions = [
|
||||
SharedOffloadRegion(
|
||||
engine_id=iid,
|
||||
num_chunks=5,
|
||||
rank=rank,
|
||||
kv_bytes_per_chunk=2 * PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE,
|
||||
num_segments=3,
|
||||
)
|
||||
for rank in range(2)
|
||||
]
|
||||
segments = regions[0]._segments
|
||||
try:
|
||||
assert [length // PAGE_SIZE for _, _, length in segments] == [4, 4, 2]
|
||||
for rank, region in enumerate(regions):
|
||||
ranges = list(region.iter_worker_slot_ranges())
|
||||
first_row = rank * 5 // 2
|
||||
assert ranges[0] == ((first_row * 2 + rank) * PAGE_SIZE, PAGE_SIZE)
|
||||
assert sorted(ranges) == [
|
||||
((row * 2 + rank) * PAGE_SIZE, PAGE_SIZE) for row in range(5)
|
||||
]
|
||||
region.create_next_worker_view(PAGE_SIZE).fill_(rank + 1)
|
||||
rows = regions[0].base_tensor.view(5, 2, PAGE_SIZE)
|
||||
assert (rows[:, 0] == 1).all() and (rows[:, 1] == 2).all()
|
||||
del rows
|
||||
finally:
|
||||
for region in regions:
|
||||
region.cleanup()
|
||||
assert not any(os.path.exists(path) for path, _, _ in segments)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Multi-worker race — concurrent construction
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -899,6 +934,7 @@ def _unmapped_registration_region(num_chunks, slot_size, num_workers):
|
||||
region.cpu_page_size = slot_size
|
||||
region._row_stride = num_workers * slot_size
|
||||
region.total_size_bytes = region.num_chunks * region._row_stride
|
||||
region._segments = [("unused", 0, region.total_size_bytes)]
|
||||
region._base = SimpleNamespace(data_ptr=lambda: 0x100000)
|
||||
region.pinned_addresses = []
|
||||
region.is_pinned = False
|
||||
|
||||
@@ -15,6 +15,8 @@ from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import
|
||||
)
|
||||
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.gpu.spec_decode.dspark.speculator import DSparkSpeculator
|
||||
|
||||
|
||||
def _markov_head(weight: torch.Tensor) -> DSparkMarkovHead:
|
||||
@@ -70,6 +72,46 @@ def test_gathered_markov_bias_matches_dense_at_full_vocab():
|
||||
torch.testing.assert_close(result, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("use_fp64", [False, True])
|
||||
def test_dspark_revisited_position_uses_step_seed(use_fp64):
|
||||
"""A cached draft at a revisited position must use its current step."""
|
||||
device = torch.device("cuda")
|
||||
batch = 4096
|
||||
spec = DSparkSpeculator.__new__(DSparkSpeculator)
|
||||
spec.acceptance_estimator = None
|
||||
spec._d2t_scatter_index = None
|
||||
spec.draft_watermarker = None
|
||||
spec.use_fp64_gumbel = use_fp64
|
||||
spec.temperature = torch.ones(batch, device=device)
|
||||
spec.seeds = torch.arange(batch, dtype=torch.int64, device=device)
|
||||
spec._step_cols = torch.arange(2, dtype=torch.int32, device=device)
|
||||
spec.draft_logits = torch.empty(batch, 2, 2, device=device)
|
||||
logits = torch.tensor([0.7, 0.3], device=device).log().expand(batch, -1)
|
||||
mapping = torch.arange(batch, device=device)
|
||||
positions = torch.full((batch,), 22, dtype=torch.int64, device=device)
|
||||
|
||||
first = spec._sample_logits(logits, mapping, positions, 1)
|
||||
repeated = spec._sample_logits(logits, mapping, positions, 1)
|
||||
next_round = spec._sample_logits(logits, mapping, positions, 0)
|
||||
torch.testing.assert_close(first, repeated)
|
||||
expected = gumbel_sample(
|
||||
logits,
|
||||
mapping,
|
||||
spec.temperature,
|
||||
spec.seeds,
|
||||
positions - 1,
|
||||
apply_temperature=True,
|
||||
is_drafting=True,
|
||||
logits_cache=spec.draft_logits,
|
||||
logits_cache_col=spec._step_cols[0],
|
||||
use_fp64=use_fp64,
|
||||
)
|
||||
torch.testing.assert_close(next_round, expected)
|
||||
mismatch_rate = (first != next_round).float().mean().item()
|
||||
assert 0.35 < mismatch_rate < 0.5
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cuda_alike_platform", [False, True])
|
||||
def test_gathered_markov_bias_dequantizes_selected_w4a16_rows(
|
||||
monkeypatch, cuda_alike_platform
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.model_executor.layers import logits_processor
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.models.lilicorr import LiLiCorrHead
|
||||
from vllm.transformers_utils.configs.eagle import EAGLEConfig
|
||||
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import CandidateSampler
|
||||
from vllm.v1.worker.gpu.spec_decode.lilicorr.speculator import LiLiCorrSpeculator
|
||||
|
||||
@@ -500,6 +501,60 @@ def test_candidate_walk_preserves_conditional_scores_and_padding(probabilistic):
|
||||
assert cache[1].isneginf().all()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("use_fp64", [False, True])
|
||||
def test_candidate_walk_revisited_position_matches_step_seed(use_fp64):
|
||||
"""The candidate walk must use the same step key as ordinary draft sampling."""
|
||||
device = torch.device("cuda")
|
||||
batch, steps, top_k = 4096, 2, 2
|
||||
sampler = CandidateSampler(batch, steps, top_k, device)
|
||||
candidates = torch.arange(top_k, device=device).expand(batch, steps, top_k)
|
||||
logits = torch.tensor([0.7, 0.3], device=device).log()
|
||||
scores = logits.expand(batch, steps, top_k, top_k)
|
||||
mapping = torch.arange(batch, device=device)
|
||||
temperature = torch.ones(batch, device=device)
|
||||
seeds = mapping.to(torch.int64)
|
||||
positions = torch.arange(21, 21 + steps, device=device).repeat(batch)
|
||||
tokens = torch.empty(batch, steps, dtype=torch.int64, device=device)
|
||||
cache = torch.empty(batch, steps, top_k, device=device)
|
||||
first_round = None
|
||||
for start in (21, 22):
|
||||
positions.view(batch, steps).copy_(
|
||||
torch.arange(start, start + steps, device=device)
|
||||
)
|
||||
sampler.sample(
|
||||
candidates,
|
||||
scores,
|
||||
batch,
|
||||
positions,
|
||||
mapping.repeat_interleave(steps),
|
||||
temperature,
|
||||
seeds,
|
||||
tokens,
|
||||
cache,
|
||||
use_fp64,
|
||||
)
|
||||
for step in range(steps):
|
||||
expected = gumbel_sample(
|
||||
logits.expand(batch, -1),
|
||||
mapping,
|
||||
temperature,
|
||||
seeds,
|
||||
positions.view(batch, steps)[:, step] - 1,
|
||||
apply_temperature=True,
|
||||
is_drafting=True,
|
||||
logits_cache=cache,
|
||||
logits_cache_col=torch.tensor(step, device=device),
|
||||
use_fp64=use_fp64,
|
||||
)
|
||||
torch.testing.assert_close(tokens[:, step], expected)
|
||||
if first_round is None:
|
||||
first_round = tokens.clone()
|
||||
else:
|
||||
mismatch_rate = (first_round[:, 1] != tokens[:, 0]).float().mean().item()
|
||||
assert 0.35 < mismatch_rate < 0.5
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("convolution", "mismatch", "quantized"),
|
||||
[
|
||||
|
||||
@@ -114,6 +114,10 @@ class _EngineReasonerStub(ParserEngineReasoningAdapter):
|
||||
def reasoning_end_token_ids(self):
|
||||
return self._end_token_ids
|
||||
|
||||
@property
|
||||
def implicit_reasoning_end_token_ids(self):
|
||||
return frozenset()
|
||||
|
||||
def find_reasoning_end(self, token_ids):
|
||||
self.windows.append(list(token_ids))
|
||||
for offset, token in enumerate(token_ids):
|
||||
|
||||
@@ -557,6 +557,79 @@ def test_gumbel_drafted_rejection_sample_is_unbiased(num_speculative_steps: int)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_block_verification", [False, True])
|
||||
@pytest.mark.parametrize("probabilistic_draft", [False, True])
|
||||
def test_multiround_rejection_sample_is_unbiased(
|
||||
use_block_verification: bool, probabilistic_draft: bool
|
||||
):
|
||||
"""Uncommitted positions must receive fresh noise on later verification rounds."""
|
||||
torch.manual_seed(42)
|
||||
num_trials, num_tokens, k = 32768, 6, 2
|
||||
device = "cuda"
|
||||
target_probs = torch.tensor(
|
||||
[0.3, 0.7] if probabilistic_draft else [0.5, 0.5], device=device
|
||||
)
|
||||
draft_logits = torch.tensor(
|
||||
[0.7, 0.3] if probabilistic_draft else [1.0, 0.0], device=device
|
||||
).log()
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_probs.log(), draft_logits, k, 1.0, num_trials
|
||||
)
|
||||
seeds = inputs["seed"]
|
||||
output = torch.full((num_trials, num_tokens + k), -1, device=device)
|
||||
output[:, 0] = torch.multinomial(target_probs, num_trials, replacement=True)
|
||||
lengths = torch.ones(num_trials, dtype=torch.int64, device=device)
|
||||
accepted = 0
|
||||
while (lengths < num_tokens).any():
|
||||
active = (lengths < num_tokens).nonzero().flatten()
|
||||
n = active.numel()
|
||||
starts = lengths[active]
|
||||
local_positions = torch.arange(k + 1, device=device).repeat(n)
|
||||
positions = (starts[:, None] + torch.arange(k + 1, device=device)).flatten()
|
||||
draft_sampled = torch.zeros((n, k + 1), dtype=torch.int64, device=device)
|
||||
draft_sampled[:, 0] = output[active, starts - 1]
|
||||
logits_cache = inputs["draft_logits"] if probabilistic_draft else None
|
||||
if probabilistic_draft:
|
||||
for step in range(k):
|
||||
draft_sampled[:, step + 1] = gumbel_sample(
|
||||
draft_logits.expand(n, -1),
|
||||
active,
|
||||
inputs["temperature"],
|
||||
seeds,
|
||||
starts + step,
|
||||
apply_temperature=True,
|
||||
is_drafting=True,
|
||||
logits_cache=logits_cache,
|
||||
logits_cache_col=torch.tensor(step, device=device),
|
||||
)
|
||||
sampled, counts = rejection_sample(
|
||||
target_logits=target_probs.log().expand(n * (k + 1), -1).contiguous(),
|
||||
draft_logits=logits_cache,
|
||||
draft_sampled=draft_sampled.flatten(),
|
||||
cu_num_logits=torch.arange(n + 1, dtype=torch.int32, device=device)
|
||||
* (k + 1),
|
||||
pos=positions,
|
||||
idx_mapping=active,
|
||||
expanded_idx_mapping=active.repeat_interleave(k + 1),
|
||||
expanded_local_pos=local_positions,
|
||||
temperature=inputs["temperature"],
|
||||
seed=seeds,
|
||||
num_speculative_steps=k,
|
||||
use_block_verification=use_block_verification,
|
||||
)
|
||||
for column in range(k + 1):
|
||||
valid = counts > column
|
||||
output[active[valid], starts[valid] + column] = sampled[valid, column]
|
||||
lengths[active] += counts
|
||||
accepted += (counts - 1).sum().item()
|
||||
|
||||
assert accepted > num_trials, "The test must exercise accepted draft tokens."
|
||||
for position in range(num_tokens):
|
||||
_assert_distribution_match(
|
||||
output[:, position], target_probs, device, label=f"position {position}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_speculative_steps", [1, 3])
|
||||
def test_greedy_rejection_sample(num_speculative_steps: int):
|
||||
"""Verify that greedy (temperature=0) always outputs the target argmax
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
import asyncio
|
||||
from collections.abc import AsyncGenerator
|
||||
from unittest import mock
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
@@ -198,3 +199,52 @@ async def test_empty_input_stream_finishes_without_engine_request():
|
||||
|
||||
assert queue.get_nowait() is STREAM_FINISHED
|
||||
llm._add_request.assert_not_awaited()
|
||||
|
||||
|
||||
def _make_streaming_llm(num_chunks: int, has_request: bool = True):
|
||||
llm = AsyncLLM.__new__(AsyncLLM)
|
||||
llm._validate_streaming_input_sampling_params = MagicMock()
|
||||
llm.get_supported_tasks = AsyncMock(return_value=("generate",))
|
||||
llm._run_output_handler = MagicMock()
|
||||
llm._add_request = AsyncMock()
|
||||
llm.model_config = MagicMock()
|
||||
llm.output_processor = MagicMock()
|
||||
llm.output_processor.has_request.return_value = has_request
|
||||
|
||||
# The first processed request is the final (end-of-stream) request.
|
||||
reqs = [MagicMock(prompt_embeds=None) for _ in range(num_chunks + 1)]
|
||||
reqs[0].request_id = "r1-int"
|
||||
llm.input_processor = MagicMock()
|
||||
llm.input_processor.process_inputs.side_effect = reqs
|
||||
return llm, reqs
|
||||
|
||||
|
||||
async def _stream_chunks(llm: AsyncLLM, num_chunks: int) -> None:
|
||||
async def chunks():
|
||||
for i in range(num_chunks):
|
||||
yield StreamingInput(prompt=f"frame-{i}")
|
||||
|
||||
with mock.patch(
|
||||
"vllm.v1.engine.async_llm.extract_prompt_components",
|
||||
return_value=("", None, None),
|
||||
):
|
||||
queue = await llm._add_streaming_input_request(
|
||||
"r1", chunks(), SamplingParams(max_tokens=4)
|
||||
)
|
||||
task = queue._input_stream_task
|
||||
assert task is not None
|
||||
await task
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("session_live", [True, False])
|
||||
async def test_handle_inputs_stops_after_session_finished(session_live: bool):
|
||||
"""Once the session's request state is gone (aborted or errored), no
|
||||
further chunks or final request are sent: they would recreate a request
|
||||
state that is never cleaned up."""
|
||||
llm, reqs = _make_streaming_llm(num_chunks=2, has_request=session_live)
|
||||
await _stream_chunks(llm, num_chunks=2)
|
||||
|
||||
sent = [call.args[0] for call in llm._add_request.await_args_list]
|
||||
# Both chunks then the final request, or only the first chunk.
|
||||
assert sent == ([reqs[1], reqs[2], reqs[0]] if session_live else [reqs[1]])
|
||||
|
||||
@@ -5,9 +5,11 @@ import unittest
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.core.kv_cache_utils as kv_cache_utils
|
||||
from tests.v1.core.utils import create_scheduler as create_spec_scheduler
|
||||
from vllm.config import DeviceConfig, VllmConfig
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
@@ -28,7 +30,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
)
|
||||
from vllm.v1.outputs import LogprobsLists, ModelRunnerOutput
|
||||
from vllm.v1.outputs import DraftTokenIds, LogprobsLists, ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
|
||||
@@ -676,3 +678,41 @@ class TestStreamingScheduler(unittest.TestCase):
|
||||
cached_state_cycle1["prompt_token_ids"]
|
||||
is not cached_state_cycle3["prompt_token_ids"]
|
||||
), "Cached states from different cycles should be independent objects."
|
||||
|
||||
|
||||
def _model_output(request_id: str, token_ids: list[int]) -> ModelRunnerOutput:
|
||||
return ModelRunnerOutput(
|
||||
req_ids=[request_id],
|
||||
req_id_to_index={request_id: 0},
|
||||
sampled_token_ids=[token_ids],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={request_id: None},
|
||||
pooler_output=[],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [False, True])
|
||||
def test_next_chunk_fully_computed_after_stop_on_accepted_draft(async_scheduling):
|
||||
"""A stop on an accepted draft trims the step's later tokens. The next
|
||||
chunk must still be computed in full rather than landing on positions
|
||||
whose KV belongs to the trimmed drafts."""
|
||||
scheduler = create_spec_scheduler(
|
||||
num_speculative_tokens=2,
|
||||
async_scheduling=async_scheduling,
|
||||
speculative_method="ngram_gpu" if async_scheduling else None,
|
||||
)
|
||||
session = DummyRequest("session", prompt_token_ids=[1, 2, 3], max_tokens=2)
|
||||
scheduler.add_request(session)
|
||||
scheduler.update_from_output(scheduler.schedule(), _model_output("session", [10]))
|
||||
scheduler.update_draft_token_ids(DraftTokenIds(["session"], [[11, 12]]))
|
||||
# Both drafts are accepted, but max_tokens stops at 11.
|
||||
scheduler.update_from_output(
|
||||
scheduler.schedule(), _model_output("session", [11, 12, 13])
|
||||
)
|
||||
assert session.status == RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
|
||||
scheduler.add_request(DummyRequest("session", prompt_token_ids=[7, 8]))
|
||||
output = scheduler.schedule()
|
||||
|
||||
assert list(session.prompt_token_ids) == [1, 2, 3, 10, 7, 8]
|
||||
assert output.num_scheduled_tokens["session"] == 2
|
||||
|
||||
@@ -6,6 +6,7 @@ from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from threading import Event
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
@@ -72,20 +73,26 @@ class MockReasoner:
|
||||
|
||||
|
||||
class MockEngineReasoner(ParserEngineReasoningAdapter):
|
||||
"""Ends reasoning implicitly on ``marker``, which is also content."""
|
||||
"""Ends reasoning implicitly on ``marker``, which is also content, and
|
||||
explicitly on ``end_marker`` when one is given."""
|
||||
|
||||
def __init__(self, tokenizer, marker: int):
|
||||
def __init__(self, tokenizer, marker: int, end_marker: int | None = None):
|
||||
self.marker = marker
|
||||
self.end_marker = end_marker
|
||||
|
||||
@property
|
||||
def reasoning_end_token_ids(self):
|
||||
return frozenset({self.marker, self.end_marker} - {None})
|
||||
|
||||
@property
|
||||
def implicit_reasoning_end_token_ids(self):
|
||||
return frozenset({self.marker})
|
||||
|
||||
def find_reasoning_end(self, token_ids):
|
||||
ids = list(token_ids)
|
||||
if self.marker in ids:
|
||||
return ReasoningEnd(ids.index(self.marker), True)
|
||||
return ReasoningEnd(len(ids), False)
|
||||
for offset, token in enumerate(token_ids):
|
||||
if token in self.reasoning_end_token_ids:
|
||||
return ReasoningEnd(offset, token == self.marker)
|
||||
return ReasoningEnd(len(token_ids), False)
|
||||
|
||||
|
||||
def _single_token(tokenizer, text: str) -> int:
|
||||
@@ -122,6 +129,7 @@ def _build_harness(
|
||||
enable_in_reasoning: bool = False,
|
||||
reasoning_parser_kwargs: dict | None = None,
|
||||
structured_outputs: StructuredOutputsParams | None = None,
|
||||
num_spec_tokens: int = NUM_SPEC_TOKENS,
|
||||
) -> tuple[StructuredOutputManager, Request]:
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(tokenizer=TOKENIZER),
|
||||
@@ -132,8 +140,10 @@ def _build_harness(
|
||||
),
|
||||
speculative_config=SpeculativeConfig(
|
||||
model="[ngram]",
|
||||
num_speculative_tokens=NUM_SPEC_TOKENS,
|
||||
),
|
||||
num_speculative_tokens=num_spec_tokens,
|
||||
)
|
||||
if num_spec_tokens
|
||||
else None,
|
||||
)
|
||||
manager = StructuredOutputManager(vllm_config)
|
||||
if use_reasoner:
|
||||
@@ -171,8 +181,20 @@ def _build_harness(
|
||||
return manager, request
|
||||
|
||||
|
||||
def _row_pattern(bitmask) -> str:
|
||||
return "".join("U" if (row == -1).all() else "C" for row in bitmask)
|
||||
def _row_pattern(bitmask, masked_ids=()) -> str:
|
||||
"""U: unconstrained; R: all but `masked_ids` allowed; C: grammar rows."""
|
||||
reasoning_row = np.full(bitmask.shape[1], -1, dtype=np.int32)
|
||||
for token_id in masked_ids:
|
||||
reasoning_row.view(np.uint32)[token_id >> 5] &= ~np.uint32(1 << (token_id & 31))
|
||||
|
||||
def kind(row) -> str:
|
||||
if (row == -1).all():
|
||||
return "U"
|
||||
if masked_ids and (row == reasoning_row).all():
|
||||
return "R"
|
||||
return "C"
|
||||
|
||||
return "".join(kind(row) for row in bitmask)
|
||||
|
||||
|
||||
def _run_real_flow(
|
||||
@@ -183,6 +205,7 @@ def _run_real_flow(
|
||||
expected_row_pattern: str,
|
||||
expected_reasoning: bool | None,
|
||||
expect_terminated: bool,
|
||||
masked_ids: list[int] | None = None,
|
||||
) -> None:
|
||||
assert len(expected_validated) <= len(raw_drafts)
|
||||
|
||||
@@ -197,7 +220,7 @@ def _run_real_flow(
|
||||
)
|
||||
assert bitmask is not None
|
||||
assert bitmask.shape[0] == len(raw_drafts) + 1
|
||||
assert _row_pattern(bitmask) == expected_row_pattern
|
||||
assert _row_pattern(bitmask, masked_ids or ()) == expected_row_pattern
|
||||
|
||||
# grammar_bitmask() must rollback any speculative state it advanced.
|
||||
assert manager.validate_tokens(request, list(raw_drafts)) == expected_validated
|
||||
@@ -375,7 +398,8 @@ FLOW_CASES = [
|
||||
FlowCase(
|
||||
raw_drafts=(" ", IMPLICIT_THINK_END, "{", "}"),
|
||||
expected_validated=(" ",),
|
||||
expected_row_pattern="UUUUU",
|
||||
# The grammar cannot start with the marker, so reasoning rows mask it.
|
||||
expected_row_pattern="RRUUU",
|
||||
expected_reasoning=False,
|
||||
expect_terminated=False,
|
||||
engine_reasoner=True,
|
||||
@@ -421,9 +445,93 @@ def test_real_flow(
|
||||
expected_row_pattern=case.expected_row_pattern,
|
||||
expected_reasoning=case.expected_reasoning,
|
||||
expect_terminated=case.expect_terminated,
|
||||
masked_ids=[reasoner_kwargs["marker"]] if case.engine_reasoner else None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", BACKENDS)
|
||||
@pytest.mark.parametrize(
|
||||
("structured_outputs", "raw_drafts", "expected_row_pattern"),
|
||||
[
|
||||
# JSON cannot start with the marker: reasoning rows mask it and keep
|
||||
# every other token, including the explicit end, allowed.
|
||||
pytest.param(
|
||||
StructuredOutputsParams(json=JSON_SCHEMA), (" ", " "), "RRR", id="json"
|
||||
),
|
||||
# The grammar can start with the marker: it stays allowed and, once
|
||||
# sampled, is the first token the grammar consumes.
|
||||
pytest.param(
|
||||
StructuredOutputsParams(regex="z[0-9]+"),
|
||||
(" ", IMPLICIT_THINK_END, "1"),
|
||||
"UUCC",
|
||||
id="regex_starts_with_marker",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_reasoning_rows_mask_grammar_invalid_implicit_end(
|
||||
tokenizer,
|
||||
backend: StructuredOutputsBackend,
|
||||
structured_outputs: StructuredOutputsParams,
|
||||
raw_drafts: tuple[str, ...],
|
||||
expected_row_pattern: str,
|
||||
):
|
||||
marker = _single_token(tokenizer, IMPLICIT_THINK_END)
|
||||
manager, request = _build_harness(
|
||||
tokenizer,
|
||||
backend,
|
||||
reasoning_ended=False,
|
||||
reasoning_parser_kwargs={
|
||||
"marker": marker,
|
||||
"end_marker": _single_token(tokenizer, THINK_END),
|
||||
},
|
||||
structured_outputs=structured_outputs,
|
||||
)
|
||||
manager.reasoner_cls = MockEngineReasoner
|
||||
drafts = _to_token_ids(tokenizer, raw_drafts)
|
||||
assert manager.validate_tokens(request, drafts) == drafts
|
||||
|
||||
bitmask = manager.grammar_bitmask(
|
||||
requests={request.request_id: request},
|
||||
structured_output_request_ids=[request.request_id],
|
||||
scheduled_spec_decode_tokens={request.request_id: drafts},
|
||||
)
|
||||
assert bitmask is not None
|
||||
assert _row_pattern(bitmask, [marker]) == expected_row_pattern
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", BACKENDS)
|
||||
def test_parallel_fill_masks_grammar_invalid_implicit_end(
|
||||
tokenizer, backend: StructuredOutputsBackend
|
||||
):
|
||||
marker, end = _to_token_ids(tokenizer, (IMPLICIT_THINK_END, THINK_END))
|
||||
manager, request = _build_harness(
|
||||
tokenizer,
|
||||
backend,
|
||||
reasoning_ended=False,
|
||||
reasoning_parser_kwargs={"marker": marker, "end_marker": end},
|
||||
num_spec_tokens=0,
|
||||
)
|
||||
manager.reasoner_cls = MockEngineReasoner
|
||||
|
||||
def row_pattern() -> str:
|
||||
bitmask = manager.grammar_bitmask(
|
||||
requests={request.request_id: request},
|
||||
structured_output_request_ids=[request.request_id],
|
||||
scheduled_spec_decode_tokens={},
|
||||
)
|
||||
assert bitmask is not None
|
||||
return _row_pattern(bitmask, [marker])
|
||||
|
||||
manager.fill_bitmask_parallel_threshold = 0
|
||||
manager.fill_bitmask_parallel_batch_size = 16
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
manager.executor_for_fillmask = executor
|
||||
assert row_pattern() == "R"
|
||||
request.append_output_token_ids([end])
|
||||
assert manager.accept_tokens(request, [end])
|
||||
assert row_pattern() == "C"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", BACKENDS)
|
||||
@pytest.mark.parametrize(
|
||||
("use_reasoner", "reasoning_ended", "enable_in_reasoning"),
|
||||
|
||||
@@ -9,12 +9,14 @@ never addressed by the logical view.
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.hisparse.binding as attn_utils_module
|
||||
import vllm.v1.hisparse.runtime as hisparse_runtime_module
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
@@ -403,49 +405,63 @@ class _FakeSharedHostRegion:
|
||||
self.cleanup_calls += 1
|
||||
|
||||
|
||||
def test_profiling_cleanup_releases_tp_shared_region_once(monkeypatch):
|
||||
"""TP-shared profiling pools must use region-aware chunk cleanup."""
|
||||
region = _FakeSharedHostRegion()
|
||||
def test_profiling_cleanup_keeps_host_pool_for_real_kv_cache(monkeypatch):
|
||||
"""The real KV cache must reuse the profiling host pool, not pin it again."""
|
||||
pinned = torch.empty(8, dtype=torch.int8)
|
||||
allocate_pinned = MagicMock(return_value=(pinned, pinned))
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module, "allocate_pinned_host_pool", allocate_pinned
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module, "check_hisparse_host_memory", lambda size: None
|
||||
)
|
||||
vllm_config = SimpleNamespace(instance_id="profiling-reuse")
|
||||
kv_cache_config = SimpleNamespace(hisparse_shared_host_pool=False)
|
||||
pool = attn_utils_module.get_hisparse_host_pool(vllm_config, kv_cache_config)
|
||||
backing = pool.allocate(8)
|
||||
runtime = SimpleNamespace(
|
||||
_host_cache=object(),
|
||||
registered_host_pool=region.base_tensor,
|
||||
hot_backing=object(),
|
||||
shared_host_region=region,
|
||||
_host_cache=backing, registered_host_pool=pinned, hot_backing=object()
|
||||
)
|
||||
forward_context = {
|
||||
"layer": SimpleNamespace(
|
||||
hisparse_cache=SimpleNamespace(runtime=runtime),
|
||||
)
|
||||
"layer": SimpleNamespace(hisparse_cache=SimpleNamespace(runtime=runtime))
|
||||
}
|
||||
released = []
|
||||
|
||||
def release_pinned_state(runtimes, pinned_host_pools, shared_host_region):
|
||||
released.append((runtimes, pinned_host_pools, shared_host_region))
|
||||
|
||||
monkeypatch.setattr(
|
||||
attn_utils_module,
|
||||
"release_pinned_state",
|
||||
release_pinned_state,
|
||||
)
|
||||
|
||||
attn_utils_module.release_hisparse_profiling_cache(forward_context)
|
||||
|
||||
assert released == [([runtime], [], region)]
|
||||
assert not hasattr(runtime, "_host_cache")
|
||||
reused = attn_utils_module.get_hisparse_host_pool(vllm_config, kv_cache_config)
|
||||
assert reused is pool
|
||||
assert reused.allocate(8) is backing
|
||||
allocate_pinned.assert_called_once_with(8)
|
||||
hisparse_runtime_module.drop_hisparse_host_pool(vllm_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure_phase", ["allocation", "binding", "buffers"])
|
||||
def test_init_hisparse_rolls_back_shared_region(monkeypatch, failure_phase):
|
||||
"""A failure after mmap allocation must not leak the shared registration."""
|
||||
@pytest.mark.parametrize("shared", [False, True])
|
||||
def test_init_hisparse_rolls_back_host_pool(monkeypatch, failure_phase, shared):
|
||||
"""Failed initialization must release shared or private registrations."""
|
||||
region = _FakeSharedHostRegion()
|
||||
registered = torch.empty(4096, dtype=torch.int8)
|
||||
cudart = MagicMock()
|
||||
cudart.cudaHostUnregister.return_value.value = 0
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
|
||||
monkeypatch.setattr(torch.cuda, "cudart", lambda: cudart)
|
||||
vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC
|
||||
),
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=1),
|
||||
instance_id="rollback",
|
||||
)
|
||||
pools = []
|
||||
|
||||
def allocate(*args):
|
||||
args[-1].shared_region = region
|
||||
pools.append(args[-1])
|
||||
if shared:
|
||||
args[-1].shared_region = region
|
||||
else:
|
||||
args[-1].registered = registered
|
||||
args[-1].backing = registered[:8]
|
||||
if failure_phase == "allocation":
|
||||
raise RuntimeError("initialization failed")
|
||||
return {}
|
||||
@@ -472,7 +488,13 @@ def test_init_hisparse_rolls_back_shared_region(monkeypatch, failure_phase):
|
||||
{},
|
||||
SimpleNamespace(),
|
||||
)
|
||||
assert region.cleanup_calls == 1
|
||||
assert region.cleanup_calls == int(shared)
|
||||
if shared:
|
||||
cudart.cudaHostUnregister.assert_not_called()
|
||||
else:
|
||||
cudart.cudaHostUnregister.assert_called_once_with(registered.data_ptr())
|
||||
assert attn_utils_module.get_hisparse_host_pool(vllm_config, None) is not pools[0]
|
||||
hisparse_runtime_module.drop_hisparse_host_pool(vllm_config)
|
||||
|
||||
|
||||
def test_reshape_padded_kv_cache_strides_by_padded_page():
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The CPU worker's automatic KV cache sizing must report the right cause.
|
||||
|
||||
A worker whose own footprint exceeds its reservation and a node that is short
|
||||
of free memory need opposite remedies, so they get separate errors.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.cpu_resource_utils import MemoryNodeInfo
|
||||
|
||||
# Importing the CPU worker patches torch and the GPU buffer utilities for the
|
||||
# whole process (vllm/v1/worker/cpu/shm.py), so the import is deferred into
|
||||
# the tests and they run only on the CPU platform.
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cpu(), reason="CPU worker test; needs the CPU platform"
|
||||
)
|
||||
|
||||
GiB = 1 << 30
|
||||
|
||||
|
||||
def _worker(monkeypatch, *, reserved, rss, total=8 * GiB, available=4 * GiB):
|
||||
import vllm.v1.worker.cpu_worker as cpu_worker_mod
|
||||
from vllm.v1.worker.cpu_worker import CPUWorker
|
||||
|
||||
worker = CPUWorker.__new__(CPUWorker)
|
||||
worker.requested_cpu_memory = reserved
|
||||
worker.cache_config = SimpleNamespace(kv_cache_memory_bytes=None)
|
||||
monkeypatch.setattr(CPUWorker, "_should_warm_up_model", lambda self: False)
|
||||
monkeypatch.setattr(
|
||||
cpu_worker_mod,
|
||||
"get_allowed_cpu_list",
|
||||
lambda: [SimpleNamespace(numa_node=0)],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
cpu_worker_mod,
|
||||
"get_memory_node_info",
|
||||
lambda node: MemoryNodeInfo(total_memory=total, available_memory=available),
|
||||
)
|
||||
process = SimpleNamespace(memory_info=lambda: SimpleNamespace(rss=rss))
|
||||
monkeypatch.setattr(cpu_worker_mod.psutil, "Process", lambda pid: process)
|
||||
return worker
|
||||
|
||||
|
||||
def test_footprint_above_reservation_asks_for_a_larger_reservation(monkeypatch):
|
||||
# 0.76 GiB already in use against a 0.16 GiB reservation (2% of 8 GiB).
|
||||
worker = _worker(monkeypatch, reserved=int(0.16 * GiB), rss=int(0.76 * GiB))
|
||||
with pytest.raises(ValueError) as exc:
|
||||
worker.determine_available_memory()
|
||||
msg = str(exc.value)
|
||||
assert "increase `--gpu-memory-utilization` to more than 0.10" in msg
|
||||
assert "--kv-cache-memory-bytes" in msg
|
||||
# Neither the misleading remedy nor a negative KV size.
|
||||
assert "other processes" not in msg
|
||||
assert "-0." not in msg
|
||||
|
||||
|
||||
def test_short_node_memory_still_asks_to_free_memory(monkeypatch):
|
||||
worker = _worker(monkeypatch, reserved=6 * GiB, rss=1 * GiB, available=2 * GiB)
|
||||
with pytest.raises(ValueError, match="Reduce CPU memory used by other processes"):
|
||||
worker.determine_available_memory()
|
||||
|
||||
|
||||
def test_budget_that_fits_sizes_the_kv_cache(monkeypatch):
|
||||
worker = _worker(monkeypatch, reserved=3 * GiB, rss=1 * GiB)
|
||||
assert worker.determine_available_memory() == 2 * GiB
|
||||
@@ -0,0 +1,217 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import gc
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization.online.lm_head import (
|
||||
quantized_lm_head_copy,
|
||||
refresh_quantized_lm_heads,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
finalize_layerwise_reload,
|
||||
initialize_layerwise_reload,
|
||||
record_metadata_for_reloading,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
# Not a multiple of the vocab padding, so the head has padded rows.
|
||||
VOCAB, HIDDEN = 4000, 512
|
||||
requires_cuda = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="Marlin needs CUDA"
|
||||
)
|
||||
# NVFP4 needs a CUDA W4A16 kernel; ROCm only runs the fp8 cases.
|
||||
requires_nvfp4 = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="nvfp4 needs a CUDA W4A16 kernel"
|
||||
)
|
||||
NVFP4 = pytest.param("nvfp4", marks=requires_nvfp4)
|
||||
|
||||
|
||||
def _target_head(monkeypatch, seed: int = 0) -> ParallelLMHead:
|
||||
# Online methods read the model dtype from the current config.
|
||||
model_config = SimpleNamespace(dtype=torch.bfloat16)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.online.fp8.get_current_vllm_config",
|
||||
lambda: SimpleNamespace(model_config=model_config),
|
||||
)
|
||||
head = ParallelLMHead(VOCAB, HIDDEN, disable_tp=True)
|
||||
gen = torch.Generator().manual_seed(seed)
|
||||
head.weight.data.copy_(torch.randn(head.weight.shape, generator=gen) * 0.02)
|
||||
return head.to("cuda", torch.bfloat16)
|
||||
|
||||
|
||||
def _tensors(module: torch.nn.Module) -> dict[str, torch.Tensor]:
|
||||
return dict(module.named_parameters()) | dict(module.named_buffers())
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@pytest.mark.parametrize(
|
||||
("quantization", "max_rel_err", "min_top1"),
|
||||
[("fp8", 0.05, 0.9), pytest.param("nvfp4", 0.15, 0.75, marks=requires_nvfp4)],
|
||||
)
|
||||
def test_tracks_bf16_head_through_logits_processor(
|
||||
default_vllm_config, dist_init, monkeypatch, quantization, max_rel_err, min_top1
|
||||
):
|
||||
head = _target_head(monkeypatch)
|
||||
target_weight = head.weight.detach().clone()
|
||||
draft_head = quantized_lm_head_copy(head, quantization)
|
||||
|
||||
assert isinstance(draft_head, ParallelLMHead)
|
||||
# The source head keeps its bf16 weight, and no bf16 copy is kept around.
|
||||
assert torch.equal(head.weight, target_weight)
|
||||
assert draft_head.weight.dtype != torch.bfloat16
|
||||
|
||||
logits_processor = LogitsProcessor(VOCAB)
|
||||
hidden = torch.randn(2048, HIDDEN, device="cuda", dtype=torch.bfloat16)
|
||||
ref = logits_processor(head, hidden).float()
|
||||
out = logits_processor(draft_head, hidden).float()
|
||||
assert out.shape == ref.shape == (2048, VOCAB)
|
||||
rel_err = ((out - ref).norm() / ref.norm()).item()
|
||||
top1 = (out.argmax(-1) == ref.argmax(-1)).float().mean().item()
|
||||
assert rel_err < max_rel_err
|
||||
assert top1 > min_top1
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@pytest.mark.parametrize("quantization", ["fp8", NVFP4])
|
||||
def test_refresh_follows_target_update(
|
||||
default_vllm_config, dist_init, monkeypatch, quantization
|
||||
):
|
||||
"""After the target's weights change, the copy is re-derived in place: it
|
||||
equals a fresh quantization of the new target, keeps its storage, and
|
||||
frees the temporary copy without waiting for the cycle collector."""
|
||||
head = _target_head(monkeypatch)
|
||||
drafter = torch.nn.Module()
|
||||
drafter.lm_head = quantized_lm_head_copy(head, quantization)
|
||||
ptrs = {k: t.data_ptr() for k, t in _tensors(drafter.lm_head).items()}
|
||||
|
||||
new_weight = _target_head(monkeypatch, seed=1).weight.detach()
|
||||
head.weight.data.copy_(new_weight)
|
||||
gc.disable()
|
||||
try:
|
||||
allocated = torch.accelerator.memory_allocated()
|
||||
refresh_quantized_lm_heads(drafter)
|
||||
assert torch.accelerator.memory_allocated() == allocated
|
||||
finally:
|
||||
gc.enable()
|
||||
|
||||
expected = _tensors(quantized_lm_head_copy(head, quantization))
|
||||
for name, tensor in _tensors(drafter.lm_head).items():
|
||||
assert tensor.data_ptr() == ptrs[name]
|
||||
if name != "workspace":
|
||||
assert torch.equal(tensor, expected[name]), name
|
||||
assert torch.equal(head.weight, new_weight)
|
||||
|
||||
|
||||
def _draft_update_worker(vllm_config, drafter: torch.nn.Module, **engine) -> Worker:
|
||||
worker = object.__new__(Worker)
|
||||
worker.vllm_config = vllm_config
|
||||
worker.weight_transfer_engine = SimpleNamespace(
|
||||
supports_draft_weight_update=True,
|
||||
reset_weight_update_target=lambda: None,
|
||||
**engine,
|
||||
)
|
||||
worker.model_runner = SimpleNamespace(get_draft_model=lambda: drafter)
|
||||
worker._weight_update_active = False
|
||||
worker._set_draft_weight_update_target = lambda: None
|
||||
return worker
|
||||
|
||||
|
||||
@requires_cuda
|
||||
def test_draft_weight_update_refreshes_copy(
|
||||
default_vllm_config, dist_init, monkeypatch
|
||||
):
|
||||
"""A draft weight update can write a target weight the drafter shares, such
|
||||
as an embedding tied to the lm_head; the copy must follow it."""
|
||||
head = _target_head(monkeypatch)
|
||||
drafter = torch.nn.Module()
|
||||
drafter.lm_head = quantized_lm_head_copy(head, "fp8")
|
||||
worker = _draft_update_worker(
|
||||
default_vllm_config,
|
||||
drafter,
|
||||
start_weight_update=lambda: None,
|
||||
finish_weight_update=lambda: None,
|
||||
)
|
||||
|
||||
worker.start_draft_weight_update()
|
||||
head.weight.data.copy_(_target_head(monkeypatch, seed=1).weight)
|
||||
worker.finish_weight_update()
|
||||
|
||||
expected = _tensors(quantized_lm_head_copy(head, "fp8"))
|
||||
for name, tensor in _tensors(drafter.lm_head).items():
|
||||
assert torch.equal(tensor, expected[name]), name
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@pytest.mark.parametrize("quantization", ["fp8", NVFP4])
|
||||
def test_draft_weight_update_loads_lm_head(
|
||||
default_vllm_config, dist_init, monkeypatch, quantization
|
||||
):
|
||||
"""A draft update carrying lm_head.weight loads the shared target head, as
|
||||
without quantization, and the copy is re-derived from it in place."""
|
||||
head = _target_head(monkeypatch)
|
||||
record_metadata_for_reloading(head)
|
||||
drafter = torch.nn.Module()
|
||||
draft_head = quantized_lm_head_copy(head, quantization)
|
||||
drafter.lm_head = draft_head
|
||||
ptrs = {k: t.data_ptr() for k, t in _tensors(draft_head).items()}
|
||||
new_weight = torch.randn(VOCAB, HIDDEN, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
def load(update_info):
|
||||
param = drafter.get_parameter("lm_head.weight")
|
||||
getattr(param, "weight_loader", default_weight_loader)(param, new_weight)
|
||||
|
||||
worker = _draft_update_worker(
|
||||
default_vllm_config,
|
||||
drafter,
|
||||
start_weight_update=lambda: initialize_layerwise_reload(drafter),
|
||||
update_weights=load,
|
||||
finish_weight_update=lambda: finalize_layerwise_reload(drafter, None),
|
||||
)
|
||||
worker.start_draft_weight_update()
|
||||
worker.update_weights({})
|
||||
worker.finish_weight_update()
|
||||
|
||||
assert drafter.lm_head is draft_head
|
||||
assert torch.equal(head.weight[:VOCAB], new_weight)
|
||||
expected = _tensors(quantized_lm_head_copy(head, quantization))
|
||||
for name, tensor in _tensors(draft_head).items():
|
||||
assert tensor.data_ptr() == ptrs[name]
|
||||
if name != "workspace":
|
||||
assert torch.equal(tensor, expected[name]), name
|
||||
hidden = torch.randn(64, HIDDEN, device="cuda", dtype=torch.bfloat16)
|
||||
assert LogitsProcessor(VOCAB)(draft_head, hidden).isfinite().all()
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@requires_nvfp4
|
||||
def test_layerwise_reload_of_drafter_keeps_copy(
|
||||
default_vllm_config, dist_init, monkeypatch
|
||||
):
|
||||
"""A drafter weight update reloads the drafter layerwise; the derived copy
|
||||
must come back unchanged, not be re-quantized."""
|
||||
head = _target_head(monkeypatch)
|
||||
drafter = torch.nn.Module()
|
||||
record_metadata_for_reloading(drafter)
|
||||
drafter.lm_head = quantized_lm_head_copy(head, "nvfp4")
|
||||
before = {k: t.clone() for k, t in _tensors(drafter.lm_head).items()}
|
||||
|
||||
def fail(*args, **kwargs):
|
||||
raise AssertionError("the copy was re-quantized")
|
||||
|
||||
monkeypatch.setattr(
|
||||
drafter.lm_head.quant_method, "process_weights_after_loading", fail
|
||||
)
|
||||
initialize_layerwise_reload(drafter)
|
||||
finalize_layerwise_reload(drafter, None)
|
||||
|
||||
after = _tensors(drafter.lm_head)
|
||||
assert after.keys() == before.keys()
|
||||
for name, tensor in before.items():
|
||||
assert torch.equal(after[name], tensor), name
|
||||
@@ -302,10 +302,8 @@ def _sync_dp(
|
||||
reduced[4] = int(allow_ubatching)
|
||||
reduced[5] = 8 # num_reqs
|
||||
|
||||
with (
|
||||
patch.object(dp_utils.dist, "all_reduce", lambda t, group: t.copy_(reduced)),
|
||||
patch.object(dp_utils, "get_dp_group", lambda: SimpleNamespace(cpu_group=None)),
|
||||
):
|
||||
dp_group = SimpleNamespace(cpu_all_reduce_=lambda t: t.copy_(reduced))
|
||||
with patch.object(dp_utils, "get_dp_group", lambda: dp_group):
|
||||
return dp_utils.sync_cudagraph_and_dp_padding(
|
||||
cudagraph_manager=cudagraph_manager,
|
||||
desired_batch_desc=BatchExecutionDescriptor(
|
||||
|
||||
@@ -65,6 +65,9 @@ class _RecordingModelRunner:
|
||||
def reset_lora_state(self) -> None:
|
||||
self.reset_lora_calls += 1
|
||||
|
||||
def get_draft_model(self) -> nn.Module | None:
|
||||
return None
|
||||
|
||||
|
||||
def _make_worker(engine: _RecordingEngine | None) -> Worker:
|
||||
worker = object.__new__(Worker)
|
||||
|
||||
@@ -62,7 +62,7 @@ def test_mrv2_kv_pool_only_wraps_backing_allocation(monkeypatch, hisparse) -> No
|
||||
|
||||
monkeypatch.setattr(
|
||||
hisparse_binding,
|
||||
"HiSparseHostPool",
|
||||
"get_hisparse_host_pool",
|
||||
lambda vllm_config, kv_cache_config: host_pool,
|
||||
)
|
||||
monkeypatch.setattr(hisparse_binding, "allocate_hisparse_kv_caches", allocate)
|
||||
|
||||
@@ -1257,6 +1257,8 @@ def test_hisparse_worker_shutdown_releases_pinned_state(monkeypatch):
|
||||
worker.cache_handles = []
|
||||
worker.pinned_host_pools = []
|
||||
worker.shared_host_region = object()
|
||||
worker.vllm_config = SimpleNamespace(instance_id="shutdown")
|
||||
pool = hisparse_runtime_module.get_hisparse_host_pool(worker.vllm_config, None)
|
||||
released = False
|
||||
|
||||
def release_pinned_state(runtimes, pinned_host_pools, shared_host_region):
|
||||
@@ -1273,6 +1275,9 @@ def test_hisparse_worker_shutdown_releases_pinned_state(monkeypatch):
|
||||
worker.shutdown()
|
||||
|
||||
assert released
|
||||
new_pool = hisparse_runtime_module.get_hisparse_host_pool(worker.vllm_config, None)
|
||||
assert new_pool is not pool
|
||||
hisparse_runtime_module.drop_hisparse_host_pool(worker.vllm_config)
|
||||
|
||||
|
||||
class _TestReplaySSMMixer(MambaMixer2):
|
||||
|
||||
Reference in New Issue
Block a user