Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-16 17:35:42 +07:00
347 changed files with 29531 additions and 3472 deletions
@@ -763,7 +763,7 @@ def test_flashinfer_cross_layer_layout(
_test_backend_correctness(
batch_spec=BATCH_SPECS[batch_spec_name],
model="meta-llama/Meta-Llama-3-8B",
model="Qwen/Qwen3-0.6B",
backend_to_test=[AttentionBackendEnum.FLASHINFER],
mask_mod=causal_mask_mod,
kv_cache_dtype=kv_cache_dtype,
@@ -786,7 +786,7 @@ def test_flashinfer_cross_layer_layout(
"single_prefill",
],
)
@pytest.mark.parametrize("model", ["meta-llama/Meta-Llama-3-8B"])
@pytest.mark.parametrize("model", ["Qwen/Qwen3-0.6B"])
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e4m3"])
def test_causal_backend_correctness(
@@ -1582,7 +1582,7 @@ def test_flashinfer_native_prefill_with_sinks(default_vllm_config):
_test_backend_correctness(
BATCH_SPECS["small_prefill"],
"meta-llama/Meta-Llama-3-8B",
"Qwen/Qwen3-0.6B",
[AttentionBackendEnum.FLASHINFER],
causal_mask_mod,
use_sinks=True,
@@ -1629,7 +1629,7 @@ def test_flashinfer_xqa_decode_correctness(default_vllm_config):
batch_spec = BATCH_SPECS["small_decode"]
vllm_config = create_vllm_config(
model_name="meta-llama/Meta-Llama-3-8B",
model_name="Qwen/Qwen3-0.6B",
max_model_len=max(batch_spec.seq_lens),
block_size=16,
)
@@ -1691,7 +1691,7 @@ def test_flashinfer_xqa_decode_correctness(default_vllm_config):
_test_backend_correctness(
batch_spec,
"meta-llama/Meta-Llama-3-8B",
"Qwen/Qwen3-0.6B",
[AttentionBackendEnum.FLASHINFER],
causal_mask_mod,
)
@@ -1994,7 +1994,7 @@ if current_platform.is_rocm():
"mixed_small",
],
)
@pytest.mark.parametrize("model", ["meta-llama/Meta-Llama-3-8B"])
@pytest.mark.parametrize("model", ["Qwen/Qwen3-0.6B"])
def test_non_causal_backend_correctness(
default_vllm_config,
batch_spec_name: str,
@@ -67,7 +67,7 @@ def test_deepseek_rocm_adaptive_builders_support_varlen_full_graphs():
DeepseekV4ROCMAiterMLASparseMetadataBuilder,
DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
v41_rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
v41_rocm.DeepseekV41ROCMAiterSparseSWAMetadataBuilder,
):
assert (
builder_cls.get_cudagraph_support(adaptive_config, SimpleNamespace())
@@ -126,7 +126,7 @@ def test_deepseek_v41_rocm_fixed_k_path_is_unchanged(
for builder_cls in (
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
v41_rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
v41_rocm.DeepseekV41ROCMAiterSparseSWAMetadataBuilder,
):
assert (
builder_cls.get_cudagraph_support(fixed_config, SimpleNamespace())
@@ -529,7 +529,7 @@ def make_builder(
window: int = WINDOW,
max_image_tokens: int = MAX_IMG,
max_num_batched_tokens: int = 64,
model_name: str = "meta-llama/Meta-Llama-3-8B",
model_name: str = "Qwen/Qwen3-0.6B",
builder_cls: type[
DeepseekSparseSWAMetadataBuilder
] = DeepseekSparseSWAMetadataBuilder,
@@ -10,9 +10,11 @@ import torch
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
import vllm.v1.attention.backends.mla.indexer as indexer
from vllm.config import CUDAGraphMode
from vllm.config import AttentionConfig, CUDAGraphMode
from vllm.model_executor.layers.sparse_attn_indexer import PrefillRowShard
MIN_ROWS = AttentionConfig().indexer_tp_shard_min_rows
def _chunk(start: int, end: int) -> SimpleNamespace:
"""Chunk over rows [start, end); bounds derive from the global row."""
@@ -80,11 +82,11 @@ def test_shard_exchange_reassembles_rows_only(
assert torch.all(buffer[decode + sum(sizes) :] == -7), "padding untouched"
@pytest.mark.parametrize("tp_size", [2, 4])
@pytest.mark.parametrize("tp_size", [2, 4, 8])
def test_balanced_row_shard_partitions_and_declines(tp_size: int) -> None:
"""The split covers every row once with >= 1 row per rank and declines
below the activation floor (or on a single rank)."""
n = indexer.MIN_TP_SHARD_ROWS_PER_RANK * tp_size
n = MIN_ROWS
shapes = [
([4 * n], [4 * n]), # fresh prompt: the full causal cost ramp
# uneven requests at different context depths
@@ -97,12 +99,38 @@ def test_balanced_row_shard_partitions_and_declines(tp_size: int) -> None:
torch.tensor(query_lens, dtype=torch.int32),
ratio,
tp_size,
n,
)
assert sizes is not None and len(sizes) == tp_size
assert sum(sizes) == sum(query_lens) and min(sizes) >= 1
lens = torch.tensor([n - 1], dtype=torch.int32)
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, tp_size) is None
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, 1) is None
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, tp_size, n) is None
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, 1, n) is None
def test_row_shard_threshold_is_total_not_per_rank() -> None:
"""The activation floor counts TOTAL rows regardless of tp_size: a
per-rank floor would need 128K rows at TP8, i.e. an unschedulable
--max-num-batched-tokens of 128K on 80 GB Hopper nodes."""
lens = torch.tensor([MIN_ROWS], dtype=torch.int32)
for tp_size in (2, 4, 8):
sizes = indexer.balanced_prefill_row_shard(lens, lens, 4, tp_size, MIN_ROWS)
assert sizes is not None
def test_tp2_old_per_rank_window_stays_replicated() -> None:
"""32K-64K total rows engaged on TP2 under the old per-rank floor
(16K rows/rank); the total floor now keeps them replicated — the
all_gatherv latency term is largest at small tp."""
window = torch.tensor([3 * MIN_ROWS // 4], dtype=torch.int32)
assert indexer.balanced_prefill_row_shard(window, window, 4, 2, MIN_ROWS) is None
def test_row_shard_floor_is_at_least_tp_size() -> None:
"""A floor below tp_size still leaves every rank at least one row."""
lens = torch.tensor([8], dtype=torch.int32)
assert indexer.balanced_prefill_row_shard(lens, lens, 4, 8, 1) is not None
assert indexer.balanced_prefill_row_shard(lens - 1, lens - 1, 4, 8, 1) is None
def test_row_sharding_gate_rejects_unsupported_configurations(
@@ -0,0 +1,149 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Equivalence tests for the fused mamba "align" block-table gather.
`mamba_get_block_table_tensor`'s align branch used to be five PyTorch
statements that expanded to seven kernel launches over a `(num_reqs,)` tensor.
It is now one Triton kernel. These tests pin the fused kernel to the exact
semantics of the expression it replaced, including cases a single-request
decode never exercises but other callers can hit.
`block_table` is deliberately tested non-contiguous: the original
`torch.gather(block_table, 1, ...)` honours any layout, and this is a shared
utility (`gdn_attn`, `mamba_attn`, `short_conv_attn` and `linear_attn` all call
it), so a column stride of 1 must not be assumed.
"""
import pytest
import torch
from vllm.v1.attention.backends.utils import mamba_get_block_table_tensor
from vllm.v1.kv_cache_interface import MambaSpec
WIDTH = 64
def _spec(block_size: int, num_speculative_blocks: int) -> MambaSpec:
# Only block_size and num_speculative_blocks are read by the align branch;
# shapes/dtypes just have to be well formed for the frozen dataclass.
return MambaSpec(
block_size=block_size,
shapes=((1,),),
dtypes=(torch.float32,),
num_speculative_blocks=num_speculative_blocks,
)
def _reference(block_table, seq_lens, block_size, num_spec_blocks):
"""The align branch exactly as it was before the fusion."""
start_indices = (seq_lens - 1) // block_size
start_indices = start_indices.clamp(min=0)
offsets = torch.arange(
1 + num_spec_blocks, device=block_table.device, dtype=torch.int32
)
indices_to_gather = (start_indices.unsqueeze(1) + offsets).to(torch.int64)
return torch.gather(block_table, 1, indices_to_gather)
def _make(nreq, block_size, nspec, device, noncontig=False):
torch.manual_seed(0)
width = WIDTH * 2 if noncontig else WIDTH
bt = torch.randint(0, 5000, (nreq, width), dtype=torch.int32, device=device)
if noncontig:
# Every other column: stride(1) == 2, which a kernel assuming a unit
# column stride would read straight through, silently returning the
# wrong blocks.
bt = bt[:, ::2]
assert not bt.is_contiguous()
# Keep start + nspec inside the row, which the gather already requires.
hi = max(1, (WIDTH - 1 - nspec) * block_size)
sl = torch.randint(0, hi, (nreq,), dtype=torch.int32, device=device)
return bt, sl
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
@pytest.mark.parametrize("nreq", [1, 2, 8, 64])
@pytest.mark.parametrize("block_size", [16, 64, 256])
@pytest.mark.parametrize("nspec", [0, 1, 2, 3, 4, 7])
def test_matches_reference(nreq, block_size, nspec):
bt, sl = _make(nreq, block_size, nspec, "cuda")
got = mamba_get_block_table_tensor(bt, sl, _spec(block_size, nspec), "align")
want = _reference(bt, sl, block_size, nspec)
assert got.shape == want.shape
assert got.dtype == want.dtype
torch.testing.assert_close(got, want, atol=0, rtol=0)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
@pytest.mark.parametrize("nspec", [0, 3, 7])
def test_non_contiguous_block_table(nspec):
"""A column stride != 1 must be honoured, not assumed away."""
bt, sl = _make(8, 64, nspec, "cuda", noncontig=True)
got = mamba_get_block_table_tensor(bt, sl, _spec(64, nspec), "align")
want = _reference(bt, sl, 64, nspec)
torch.testing.assert_close(got, want, atol=0, rtol=0)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
def test_zero_seq_len_clamps_to_first_row():
"""CUDA-graph padding rows carry seq_len 0; the old code clamped to row 0."""
bt = torch.randint(0, 5000, (4, WIDTH), dtype=torch.int32, device="cuda")
sl = torch.zeros(4, dtype=torch.int32, device="cuda")
got = mamba_get_block_table_tensor(bt, sl, _spec(64, 3), "align")
torch.testing.assert_close(got, _reference(bt, sl, 64, 3), atol=0, rtol=0)
# start clamps to 0, so this is just the first 1 + nspec columns.
torch.testing.assert_close(got, bt[:, :4], atol=0, rtol=0)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
def test_zero_requests_returns_empty():
"""An empty batch must not launch a (0,) grid."""
bt = torch.zeros((0, WIDTH), dtype=torch.int32, device="cuda")
sl = torch.zeros(0, dtype=torch.int32, device="cuda")
got = mamba_get_block_table_tensor(bt, sl, _spec(64, 3), "align")
assert got.shape == (0, 4)
assert got.dtype == bt.dtype
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
def test_none_mode_returns_the_input_unchanged():
"""`MambaCacheMode` is Literal["align", "none"]; only "none" passes through.
An earlier version of this test also parametrised a "mode" of "all", which
is not a member of that Literal. Anything that is not "none" takes the
align branch and returns a NEW tensor, so asserting identity for it failed
-- correctly. The passthrough contract applies to "none" alone.
"""
bt = torch.randint(0, 5000, (4, WIDTH), dtype=torch.int32, device="cuda")
sl = torch.randint(0, 1000, (4,), dtype=torch.int32, device="cuda")
assert mamba_get_block_table_tensor(bt, sl, _spec(64, 3), "none") is bt
# ---------------------------------------------------------------------------
# CPU tensors. Every test above runs on "cuda", which is exactly how the
# regression reached review: `mamba_get_block_table_tensor` is a SHARED
# utility and its callers legitimately pass CPU tensors -- the GDN and
# linear-attention metadata builders are unit-tested with
# `DEVICE = torch.device("cpu")`. A Triton kernel cannot run on those at all,
# while the torch expression it replaced could. These run without a GPU.
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("nspec", [0, 1, 3])
@pytest.mark.parametrize("block_size", [1, 16, 64])
def test_cpu_tensors_match_the_reference(nspec, block_size):
"""The align branch must work on CPU, not just on the accelerator."""
width = 2 + nspec + 8
bt = torch.arange(5 * width, dtype=torch.int32).reshape(5, width)
sl = torch.tensor([0, 1, block_size, block_size * 3 + 1, 7], dtype=torch.int32)
got = mamba_get_block_table_tensor(bt, sl, _spec(block_size, nspec), "align")
want = _reference(bt, sl, block_size, nspec)
assert torch.equal(got, want), f"cpu mismatch: {got.tolist()} != {want.tolist()}"
assert got.device == bt.device
assert got.dtype == bt.dtype
def test_cpu_none_mode_passes_through():
bt = torch.arange(12, dtype=torch.int32).reshape(3, 4)
sl = torch.tensor([1, 2, 3], dtype=torch.int32)
assert mamba_get_block_table_tensor(bt, sl, _spec(16, 3), "none") is bt
@@ -171,6 +171,11 @@ def test_paged_prefill_indices_match_gathered_prefill(
from vllm.v1.kv_cache_interface import SlidingWindowMLASpec
rocm = importlib.import_module(f"vllm.models.{model}.amd.rocm")
builder_cls = (
rocm.DeepseekV41ROCMAiterSparseSWAMetadataBuilder
if model == "deepseek_v41"
else rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder
)
device = torch.device("cuda")
gen = torch.Generator().manual_seed(compress_ratio)
window, width, block_size, max_model_len = 16, 32, 16, 128
@@ -233,9 +238,7 @@ def test_paged_prefill_indices_match_gathered_prefill(
monkeypatch.setattr(
rocm_aiter_ops, "is_triton_sparse_mla_enabled", lambda: enabled
)
return rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder(
swa_spec, ["swa"], vllm_config, device
).build(0, common)
return builder_cls(swa_spec, ["swa"], vllm_config, device).build(0, common)
assert build(enabled=False).prefill_swa_ragged_indices is None
md = build(enabled=True)
@@ -8,6 +8,8 @@ from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm.config.attention import AttentionConfig
from vllm.config.parallel import ParallelConfig
from vllm.platforms import current_platform
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.selector import AttentionSelectorConfig
@@ -44,6 +46,48 @@ def cleared_attention_selector_cache():
_cached_get_attn_backend.cache_clear()
def test_deepseek_rocm_backend_registry_paths_are_model_specific():
assert AttentionBackendEnum.ROCM_FLASHMLA_SPARSE_DSV4.get_path() == (
"vllm.models.deepseek_v4.amd.rocm.DeepseekV4ROCMAiterMLASparseBackend"
)
assert AttentionBackendEnum.ROCM_FLASHMLA_SPARSE_DSV41.get_path() == (
"vllm.models.deepseek_v41.amd.rocm.DeepseekV41ROCMAiterMLASparseBackend"
)
@pytest.mark.parametrize(
"backend_name, expect_warning",
[
("ROCM_FLASHMLA_SPARSE_DSV4", True),
("ROCM_FLASHMLA_SPARSE_DSV41", False),
],
ids=["legacy", "canonical"],
)
def test_deepseek_v41_rocm_accepts_legacy_backend_alias(backend_name, expect_warning):
from vllm.models.deepseek_v41.amd import model as dsv41_model
from vllm.models.deepseek_v41.amd.rocm import (
DeepseekV41ROCMAiterMLAAttention,
DeepseekV41ROCMAiterMLASparseBackend,
)
config = SimpleNamespace(
attention_config=AttentionConfig(backend=backend_name),
parallel_config=ParallelConfig(),
)
with patch.object(dsv41_model.logger, "warning_once") as warning_once:
selected = dsv41_model._select_dsv4_attn_cls(config)
assert selected is DeepseekV41ROCMAiterMLAAttention
if expect_warning:
warning_once.assert_called_once()
else:
warning_once.assert_not_called()
assert (
DeepseekV41ROCMAiterMLASparseBackend.get_name() == "ROCM_FLASHMLA_SPARSE_DSV41"
)
def test_aiter_unified_attention_uses_dedicated_metadata_builder():
from vllm.v1.attention.backends.rocm_aiter_unified_attn import (
RocmAiterUnifiedAttentionBackend,
+1 -1
View File
@@ -180,7 +180,7 @@ def create_standard_kv_cache_spec(
def create_vllm_config(
model_name: str = "meta-llama/Meta-Llama-3-8B",
model_name: str = "Qwen/Qwen3-0.6B",
tensor_parallel_size: int = 1,
max_model_len: int = 1024,
dtype: ModelDType | torch.dtype = "auto",
+18
View File
@@ -52,6 +52,7 @@ from vllm.v1.core.kv_cache_utils import (
hash_block_tokens,
init_none_hash,
make_block_hash_with_group_id,
null_head_len,
tensor_data,
to_event_extra_keys,
)
@@ -2156,3 +2157,20 @@ def test_wrapped_mamba_group_requires_block_zeroing():
)
assert config.needs_kv_cache_zeroing
def test_null_head_len():
null = KVCacheBlock(0, is_null=True)
real = [KVCacheBlock(i) for i in range(1, 4)]
tables = [
[],
real,
[null] * 5,
[null] * 5 + real,
[null] * 2 + real[:1] + [null] * 3 + real[1:], # not one contiguous head
]
for table in tables:
expected = next((i for i, b in enumerate(table) if not b.is_null), len(table))
assert null_head_len(table) == expected
blocks = KVCacheBlocks(([null] * 4 + real, [null] + real[:1] + [null]))
assert blocks.get_unhashed_block_ids_all_groups() == [[1, 2, 3], [1]]
+283
View File
@@ -558,6 +558,249 @@ def test_schedule_prefills_gating(has_running: bool):
assert any(r.req_id == "new0" for r in output.scheduled_new_reqs)
def _decode_one(scheduler, output, req_ids: list[str]) -> None:
"""Feed back one sampled token per request, so they keep decoding."""
scheduler.update_from_output(
output,
ModelRunnerOutput(
req_ids=req_ids,
req_id_to_index={req_id: i for i, req_id in enumerate(req_ids)},
sampled_token_ids=[[0] for _ in req_ids],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
),
)
def _start_decoding(scheduler, req_id: str = "run0"):
"""Prefill one request and take it into the decode state."""
(request,) = create_requests(
num_requests=1, num_tokens=8, max_tokens=100, req_ids=[req_id]
)
scheduler.add_request(request)
output = scheduler.schedule()
assert req_id in output.num_scheduled_tokens
_decode_one(scheduler, output, [req_id])
assert len(scheduler.running) == 1
return output
@pytest.mark.parametrize("interval", [2, 4])
def test_local_prefill_interval_holds_prefill_between_bursts(interval: int):
"""A single engine applies `prefill_schedule_interval` in the scheduler,
timed from the last prefill-carrying step: a waiting prefill is held for the
remainder of the interval while running requests keep decoding, then
admitted once the interval has elapsed.
"""
scheduler = create_scheduler(prefill_schedule_interval=interval)
# This prefill starts the interval.
_start_decoding(scheduler)
assert scheduler.last_prefill_step == scheduler.num_executed_steps
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
scheduler.add_request(new_req)
# Steps inside the interval decode alone; the waiting prefill is held.
for _ in range(interval - 1):
output = scheduler.schedule()
assert "new0" not in output.num_scheduled_tokens
assert "run0" in output.num_scheduled_tokens
assert new_req.status == RequestStatus.WAITING
_decode_one(scheduler, output, ["run0"])
# The interval has now elapsed, so prefill is admitted again.
output = scheduler.schedule()
assert "new0" in output.num_scheduled_tokens
assert scheduler.last_prefill_step == scheduler.num_executed_steps
def test_empty_schedule_does_not_advance_the_prefill_cadence():
"""An empty schedule() must not consume interval budget.
`current_step` counts every schedule() call. With async scheduling,
step_with_batch_queue() calls schedule() again whenever the batch queue has
room and has_requests() is true -- so a call lands while the previous batch
is still in flight and nothing new can be scheduled. An async KV connector
makes this common, since requests sit in lookup or WAITING_FOR_REMOTE_KVS.
Timing the cadence on current_step lets those empty calls expire the hold
early. Reported on #54627 against the LMCache MP connector.
"""
scheduler = create_scheduler(prefill_schedule_interval=4)
_start_decoding(scheduler)
# One real decode step, left in flight (no update_from_output yet).
output = scheduler.schedule()
assert output.total_num_scheduled_tokens > 0
executed_before = scheduler.num_executed_steps
step_before = scheduler.current_step
# Re-entering schedule() while that batch is still unaccounted for
# schedules nothing -- this is the async-scheduling shape.
for _ in range(3):
empty = scheduler.schedule()
assert empty.total_num_scheduled_tokens == 0
assert scheduler.current_step == step_before + 3, "current_step should advance"
assert scheduler.num_executed_steps == executed_before, (
"an empty schedule() must not count as an executed step"
)
def test_prefill_cadence_counts_executed_steps_not_schedule_calls():
"""The hold spans `interval` EXECUTED steps, however many empty schedule()
calls are interleaved.
Counting schedule() calls instead lets each empty call eat interval budget,
so with an async KV connector prefill lands on nearly every step and the
feature stops doing anything.
"""
interval = 4
scheduler = create_scheduler(prefill_schedule_interval=interval)
_start_decoding(scheduler)
prefill_at = scheduler.num_executed_steps
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
scheduler.add_request(new_req)
empty_calls = 0
admitted_at = None
for _ in range(50):
output = scheduler.schedule()
if "new0" in output.num_scheduled_tokens:
admitted_at = scheduler.num_executed_steps
break
if output.total_num_scheduled_tokens == 0:
empty_calls += 1
continue
# A real decode step. Leave it in flight and re-enter schedule() twice,
# as async scheduling does while the batch queue has room.
for _ in range(2):
extra = scheduler.schedule()
if "new0" in extra.num_scheduled_tokens:
admitted_at = scheduler.num_executed_steps
break
empty_calls += 1
if admitted_at is not None:
break
_decode_one(scheduler, output, ["run0"])
assert admitted_at is not None, "prefill was never admitted"
assert empty_calls > 0, "test never exercised an empty schedule() call"
# The hold is measured in executed steps. Counting schedule() calls would
# admit after far fewer, because the empty re-entries above would count.
assert admitted_at - prefill_at == interval, (
f"hold lasted {admitted_at - prefill_at} executed steps, expected "
f"{interval}; empty schedule() calls were counted against the cadence"
)
assert scheduler.current_step > scheduler.num_executed_steps
def test_local_prefill_interval_defaults_to_no_gating():
"""The default interval of 1 leaves admission exactly as it was: a waiting
prefill is admitted on the very next step.
"""
scheduler = create_scheduler()
assert scheduler.local_prefill_interval == 1
_start_decoding(scheduler)
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
scheduler.add_request(new_req)
output = scheduler.schedule()
assert "new0" in output.num_scheduled_tokens
def test_local_prefill_interval_left_to_engine_under_dp():
"""Under data parallelism the cadence must stay aligned across ranks, so the
DP engine core drives it and the scheduler must not gate on its own.
"""
scheduler = create_scheduler(prefill_schedule_interval=4, data_parallel_size=2)
assert scheduler.local_prefill_interval == 1
_start_decoding(scheduler)
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
scheduler.add_request(new_req)
output = scheduler.schedule()
assert "new0" in output.num_scheduled_tokens
def test_local_prefill_interval_never_idles_the_model():
"""With nothing running to decode in prefill's place, the interval must not
hold prefill back, or the step would be wasted entirely.
"""
scheduler = create_scheduler(prefill_schedule_interval=8)
_start_decoding(scheduler)
# Finish the only running request, leaving nothing to decode.
scheduler.finish_requests("run0", RequestStatus.FINISHED_ABORTED)
assert not scheduler.running
(new_req,) = create_requests(num_requests=1, num_tokens=8, req_ids=["new0"])
scheduler.add_request(new_req)
output = scheduler.schedule()
assert "new0" in output.num_scheduled_tokens
@pytest.mark.parametrize("interval", [2, 3])
def test_local_prefill_interval_defers_inflight_prefill_chunk(interval: int):
"""The single-engine interval gates prefill WORK, not just new admissions.
A request that is already partially prefilled is held for the remainder of
the interval too, exactly as `throttle_prefills` does for DP. That is the
point of the cadence -- a held step must carry no prefill at all, or decode
does not get the uninterrupted stretch the interval exists to give it.
"""
scheduler = create_scheduler(
max_num_seqs=16,
max_num_batched_tokens=50,
enable_chunked_prefill=True,
prefill_schedule_interval=interval,
)
# A short request that finishes prefill in one step -> a running decode.
(decode_req,) = create_requests(num_requests=1, num_tokens=4, req_ids=["dec0"])
scheduler.add_request(decode_req)
output = scheduler.schedule()
_decode_one(scheduler, output, ["dec0"])
assert decode_req in scheduler.running and not decode_req.is_prefill_chunk
# A long request (80 tokens, budget 50) -> prefilled in chunks. That first
# prefill closed the interval, so walk to the next step that opens it.
(chunk_req,) = create_requests(num_requests=1, num_tokens=80, req_ids=["chk0"])
scheduler.add_request(chunk_req)
for _ in range(interval - 1):
output = scheduler.schedule()
assert "chk0" not in output.num_scheduled_tokens
_decode_one(scheduler, output, ["dec0"])
output = scheduler.schedule()
assert output.num_scheduled_tokens["chk0"] > 0
scheduler.update_from_output(
output,
ModelRunnerOutput(
req_ids=["dec0", "chk0"],
req_id_to_index={"dec0": 0, "chk0": 1},
sampled_token_ids=[[0], []], # no token sampled for partial prefill
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
),
)
assert chunk_req.is_prefill_chunk # still mid-prefill, in running
# Held steps: the in-flight chunk is deferred along with new admissions,
# while the decode keeps running.
for _ in range(interval - 1):
output = scheduler.schedule()
assert "chk0" not in output.num_scheduled_tokens
assert "dec0" in output.num_scheduled_tokens
_decode_one(scheduler, output, ["dec0"])
# Once the interval has elapsed the chunk resumes.
output = scheduler.schedule()
assert "chk0" in output.num_scheduled_tokens
def _setup_remote_kv_resume(num_prompt_tokens: int, matched_tokens: int):
"""Drive a remote-KV request `r2` to the resume point (async load complete)
while another request `r1` is already decoding, so the step is throttle-
@@ -1916,6 +2159,46 @@ def test_draft_slots_budgeted_per_scheduled_request(tmp_path, monkeypatch):
assert scheduler.schedule().num_scheduled_tokens == {"0": 10, "1": 4}
@pytest.mark.parametrize(("method", "max_num_reqs"), [("dflash", 4), ("dspark", 5)])
def test_block_drafters_cap_requests_instead_of_draft_slots(
tmp_path, method, max_num_reqs
):
(tmp_path / "config.json").write_text(
'{"architectures": ["OPTForCausalLM"], "model_type": "opt"}'
)
base = create_scheduler(
model=str(tmp_path),
max_num_batched_tokens=20,
num_speculative_tokens=4,
parallel_drafting=True,
skip_tokenizer_init=True,
)
# DFlash/DSpark need a resolvable draft model, so switch the ngram config in
# place and redo the config step. DSpark then has 4 lookahead tokens, not 5.
vllm_config = base.vllm_config
object.__setattr__(vllm_config.speculative_config, "method", method)
object.__setattr__(
vllm_config.speculative_config,
"draft_model_config",
SimpleNamespace(hf_config=SimpleNamespace()),
)
vllm_config._set_max_num_scheduled_tokens()
scheduler = Scheduler(
vllm_config=vllm_config,
kv_cache_config=base.kv_cache_config,
structured_output_manager=base.structured_output_manager,
block_size=base.block_size,
)
for request in create_requests(num_requests=8, num_tokens=2):
scheduler.add_request(request)
# Admission stops at 20 // num_lookahead_tokens, and no draft slots are
# charged: each admitted prompt is scheduled in full.
assert scheduler.schedule().num_scheduled_tokens == {
str(i): 2 for i in range(max_num_reqs)
}
# Note - these test cases mirror some of those in test_rejection_sampler.py
@pytest.mark.parametrize(
"spec_tokens,output_tokens,expected,expected_per_req",
@@ -1013,6 +1013,7 @@ def test_predictor_matches_allocator_blocks_calculation_with_admission_cap():
)
total_computed = num_tokens
def test_mamba_honors_drop_eagle_block():
"""MambaManager must drop the EAGLE lookahead block, like FullAttentionManager.
@@ -1067,7 +1068,7 @@ def test_mamba_honors_drop_eagle_block():
kv_cache_spec=spec,
drop_eagle_block=drop_eagle_block,
alignment_tokens=block_size,
)[0]
)[0][0]
)
# Control: with no eagle drop, both groups see the whole cached prefix.
@@ -1082,3 +1083,35 @@ def test_mamba_honors_drop_eagle_block():
f"MambaManager ignored drop_eagle_block: full-attention dropped the "
f"lookahead block (hit={full_hit}), mamba kept it (hit={mamba_hit})"
)
def test_pop_blocks_for_free_trims_null_head():
block_size = 2
spec = SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=4,
)
block_pool = BlockPool(
num_gpu_blocks=10, enable_caching=True, hash_block_size=block_size
)
manager = get_sliding_window_manager(spec, block_pool)
null = block_pool.null_block
real = block_pool.get_new_blocks(2)
manager.req_to_blocks["swa"] = [null] * 3 + real
manager.req_to_blocks["dense"] = dense = block_pool.get_new_blocks(2)
assert manager.pop_blocks_for_free("swa") == real
assert manager.pop_blocks_for_free("dense") is dense # no null head, no copy
assert "swa" not in manager.req_to_blocks
assert all(b.ref_cnt == 1 for b in real + dense)
# The null block is never pooled, so touch / free_blocks leave it alone.
null_ref_cnt = null.ref_cnt
num_free_blocks = block_pool.get_num_free_blocks()
block_pool.touch([null])
block_pool.free_blocks([null])
assert null.ref_cnt == null_ref_cnt
assert block_pool.get_num_free_blocks() == num_free_blocks
+2
View File
@@ -85,6 +85,7 @@ def create_scheduler(
kv_cache_spec: KVCacheSpec | None = None,
runner: RunnerOption = "auto",
per_request_spec_decode_metrics: str = "none",
prefill_schedule_interval: int = 1,
scheduling_policy: SchedulerPolicy = "fcfs",
diffusion_canvas_length: int | None = None,
scheduler_cls: type[Scheduler] | None = None,
@@ -134,6 +135,7 @@ def create_scheduler(
enable_chunked_prefill=enable_chunked_prefill,
async_scheduling=async_scheduling,
is_encoder_decoder=model_config.is_encoder_decoder,
prefill_schedule_interval=prefill_schedule_interval,
# Ensure admission/preemption mechanics are deterministic
watermark=0.0,
policy=scheduling_policy,
@@ -160,6 +160,8 @@ def _make_manager_with_budgets(budgets: list[int]) -> EncoderCudaGraphManager:
Skips the parts of __init__ that require a real VllmConfig / model
by patching the attributes directly after construction.
"""
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig
mgr = object.__new__(EncoderCudaGraphManager)
mgr.token_budgets = sorted(budgets)
mgr.path_token_budgets = {"default": mgr.token_budgets}
@@ -1044,6 +1046,278 @@ class TestGetInputModality:
assert model.get_input_modality(mm_kwargs) == "video"
# ---------------------------------------------------------------------------
# GPU tests — DeepSeek-VL2 batched-tile encoder CUDA graph
# ---------------------------------------------------------------------------
# DeepSeek-VL2 mock constants (kept small for fast capture)
_DSV2_H = 2 # projector output side per tile (h = w)
_DSV2_DIM = 16 # hidden size
_DSV2_IMG_SIZE = 4 # each tile is 4×4 pixels
_DSV2_BUDGETS = [16, 64]
_DSV2_MAX_BATCH = 4
def _dsv2_output_tokens(tw: int, th: int) -> int:
"""Expected output tokens for an image with (tw, th) local tiles."""
h = w = _DSV2_H
return h * (w + 1) + th * h * (tw * w + 1) + 1
def _make_deepseek_vl2_mm_kwargs(
spatial_crops: list[list[int]],
device: torch.device,
dtype: torch.dtype,
) -> dict[str, Any]:
"""spatial_crops: [[tw, th], ...] per image."""
total_tiles = sum(1 + int(tw) * int(th) for tw, th in spatial_crops)
pixel_values = torch.randn(
total_tiles, 3, _DSV2_IMG_SIZE, _DSV2_IMG_SIZE, device=device, dtype=dtype
)
return {
"pixel_values": pixel_values,
"images_spatial_crop": torch.tensor(
spatial_crops, dtype=torch.long, device=device
),
}
class MockDeepseekVL2Model(torch.nn.Module, SupportsEncoderCudaGraph):
"""Mock of DeepseekVLV2ForCausalLM for CUDA graph tests.
Simulates the batched-tile ViT pattern: pixel_values is
[N_tiles, 3, H, W], output is assembled with newlines and
view-separator in postprocess_encoder_output.
"""
def __init__(self):
super().__init__()
hw = _DSV2_H * _DSV2_H
in_features = 3 * _DSV2_IMG_SIZE * _DSV2_IMG_SIZE
self.proj = torch.nn.Linear(in_features, hw * _DSV2_DIM)
self.image_newline = torch.nn.Parameter(torch.randn(_DSV2_DIM))
self.view_seperator = torch.nn.Parameter(torch.randn(_DSV2_DIM))
self.global_view_pos = "head"
def get_max_frames_per_video(self) -> int:
return 1
def get_encoder_cudagraph_config(self) -> EncoderCudaGraphConfig:
return EncoderCudaGraphConfig(
modalities=["image"],
buffer_keys=["pixel_values"],
out_hidden_size=_DSV2_DIM,
)
def get_encoder_cudagraph_budget_range(self, vllm_config) -> tuple[int, int]:
return (_dsv2_output_tokens(1, 1), 128)
def get_encoder_cudagraph_item_specs(
self, mm_kwargs: dict[str, Any]
) -> list[EncoderItemSpec]:
h = w = _DSV2_H
specs = []
for row in mm_kwargs["images_spatial_crop"].tolist():
tw, th = int(row[0]), int(row[1])
if tw == 0 or th == 0:
break
specs.append(
EncoderItemSpec(
input_size=1 + tw * th,
output_tokens=h * (w + 1) + th * h * (tw * w + 1) + 1,
)
)
return specs
def select_encoder_cudagraph_items(
self, mm_kwargs: dict[str, Any], indices: list[int]
) -> dict[str, Any]:
pixel_values = mm_kwargs["pixel_values"]
images_spatial_crop = mm_kwargs["images_spatial_crop"]
if not indices:
return {
"pixel_values": pixel_values[:0],
"images_spatial_crop": images_spatial_crop[:0],
}
tiles_per_image = [
1 + int(row[0]) * int(row[1]) for row in images_spatial_crop.tolist()
]
cum = [0]
for t in tiles_per_image:
cum.append(cum[-1] + t)
selected_pv = torch.cat([pixel_values[cum[i] : cum[i + 1]] for i in indices])
return {
"pixel_values": selected_pv,
"images_spatial_crop": images_spatial_crop[list(indices)],
}
def prepare_encoder_cudagraph_capture_inputs(
self,
token_budget: int,
max_batch_size: int,
max_frames_per_batch: int,
device: torch.device,
dtype: torch.dtype,
path: str = "default",
axis_keys: tuple[Hashable, ...] | None = None,
) -> EncoderCudaGraphCaptureInputs:
hw = _DSV2_H * _DSV2_H
max_tiles = (token_budget + hw - 1) // hw
dummy = torch.randn(
max_tiles, 3, _DSV2_IMG_SIZE, _DSV2_IMG_SIZE, device=device, dtype=dtype
)
return EncoderCudaGraphCaptureInputs(values={"pixel_values": dummy})
def prepare_encoder_cudagraph_replay_buffers(
self,
mm_kwargs: dict[str, Any],
max_batch_size: int,
max_frames_per_batch: int,
path: str = "default",
) -> EncoderCudaGraphReplayBuffers:
return EncoderCudaGraphReplayBuffers(
values={"pixel_values": mm_kwargs["pixel_values"]}
)
def _tile_forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
"""Project tiles: [N, 3, H, W] → [N, hw, DIM]."""
n = pixel_values.shape[0]
flat = pixel_values.view(n, -1)
hw = _DSV2_H * _DSV2_H
return self.proj(flat).view(n, hw, _DSV2_DIM)
def encoder_cudagraph_forward(
self, values: dict[str, torch.Tensor], path: str = "default"
) -> torch.Tensor:
return self._tile_forward(values["pixel_values"])
def encoder_eager_forward(
self, mm_kwargs: dict[str, Any], path: str = "default"
) -> torch.Tensor:
# Manager always runs `postprocess_encoder_output` after this, whether
# the batch went through eager or CUDA graph replay, so this must
# return the same raw per-tile shape as `encoder_cudagraph_forward`.
return self._tile_forward(mm_kwargs["pixel_values"])
def postprocess_encoder_output(
self,
outputs: dict[str, torch.Tensor],
indices: list[int],
per_item_out_tokens: list[int],
dest: dict[int, torch.Tensor] | list[torch.Tensor | None],
clone: bool = False,
batch_mm_kwargs: dict[str, Any] | None = None,
) -> None:
assert batch_mm_kwargs is not None
output = outputs["default"]
images_spatial_crop = batch_mm_kwargs["images_spatial_crop"]
_, hw, n_dim = output.shape
h = w = int(hw**0.5)
tile_offset = 0
for rank, img_idx in enumerate(indices):
tw = int(images_spatial_crop[rank][0])
th = int(images_spatial_crop[rank][1])
n_tiles = 1 + tw * th
tiles = output[tile_offset : tile_offset + n_tiles]
tile_offset += n_tiles
global_f = tiles[0].view(h, w, n_dim)
newline_g = self.image_newline.view(1, 1, n_dim).expand(h, 1, n_dim)
global_f = torch.cat([global_f, newline_g], dim=1).view(-1, n_dim)
local_f = tiles[1:].view(th, tw, h, w, n_dim)
local_f = local_f.permute(0, 2, 1, 3, 4).reshape(th * h, tw * w, n_dim)
newline_l = self.image_newline.view(1, 1, n_dim).expand(th * h, 1, n_dim)
local_f = torch.cat([local_f, newline_l], dim=1).view(-1, n_dim)
if self.global_view_pos == "head":
emb = torch.cat([global_f, self.view_seperator[None], local_f])
else:
emb = torch.cat([local_f, self.view_seperator[None], global_f])
if isinstance(dest, dict):
dest[img_idx] = emb.clone() if clone else emb
else:
dest[rank] = emb.clone() if clone else emb
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
class TestDeepseekVL2CudaGraph:
"""CUDA graph tests for the batched-tile encoder pattern (DeepSeek-VL2)."""
def setup_method(self):
self.device = torch.device("cuda:0")
self.dtype = torch.float16
self.model = MockDeepseekVL2Model().to(self.device).half()
self.mgr = _make_manager_for_gpu(
self.model, _DSV2_BUDGETS, _DSV2_MAX_BATCH, self.device, self.dtype
)
self.graph_pool = current_platform.graph_pool_handle()
self.mgr.capture(graph_pool=self.graph_pool)
def test_capture_creates_one_graph_per_budget(self):
assert set(self.mgr.budget_graphs["default"].keys()) == set(_DSV2_BUDGETS)
def test_execute_returns_one_tensor_per_image(self):
mm_kwargs = _make_deepseek_vl2_mm_kwargs(
[[1, 1], [1, 1]], self.device, self.dtype
)
result = self.mgr.execute(mm_kwargs)
assert result is not None
assert len(result) == 2
def test_output_tokens_1x1_tile(self):
# global=h*(w+1)=2*3=6, local=1*2*(1*2+1)=6, sep=1 → 13
expected = _dsv2_output_tokens(1, 1)
mm_kwargs = _make_deepseek_vl2_mm_kwargs([[1, 1]], self.device, self.dtype)
result = self.mgr.execute(mm_kwargs)
assert result is not None
assert result[0].shape == (expected, _DSV2_DIM)
def test_output_tokens_2x1_tile(self):
# global=6, local=1*2*(2*2+1)=10, sep=1 → 17
expected = _dsv2_output_tokens(2, 1)
mm_kwargs = _make_deepseek_vl2_mm_kwargs([[2, 1]], self.device, self.dtype)
result = self.mgr.execute(mm_kwargs)
assert result is not None
assert result[0].shape == (expected, _DSV2_DIM)
def test_output_tokens_multi_image(self):
# Image 0: tw=1,th=1 → 13; Image 1: tw=2,th=1 → 17
e0 = _dsv2_output_tokens(1, 1)
e1 = _dsv2_output_tokens(2, 1)
mm_kwargs = _make_deepseek_vl2_mm_kwargs(
[[1, 1], [2, 1]], self.device, self.dtype
)
result = self.mgr.execute(mm_kwargs)
assert result is not None
assert result[0].shape == (e0, _DSV2_DIM)
assert result[1].shape == (e1, _DSV2_DIM)
def test_eager_fallback_when_tokens_exceed_all_budgets(self):
# tw=4,th=4: global=6, local=4*2*(4*2+1)=72, sep=1 → 79 > max budget 64
tw, th = 4, 4
expected = _dsv2_output_tokens(tw, th)
assert expected > _DSV2_BUDGETS[-1], "test precondition: must exceed max budget"
mm_kwargs = _make_deepseek_vl2_mm_kwargs([[tw, th]], self.device, self.dtype)
result = self.mgr.execute(mm_kwargs)
assert result is not None
assert len(result) == 1
assert result[0].shape == (expected, _DSV2_DIM)
assert self.mgr.graph_misses >= 1
def test_graph_hit_counter(self):
mm_kwargs = _make_deepseek_vl2_mm_kwargs(
[[1, 1], [1, 1]], self.device, self.dtype
)
self.mgr.execute(mm_kwargs)
assert self.mgr.graph_hits == 2
# ---------------------------------------------------------------------------
# GPU tests — video capture, replay, fallback, and mixed image+video
# ---------------------------------------------------------------------------
+3 -1
View File
@@ -368,9 +368,11 @@ async def test_dp_pause_late_request_does_not_block_drain():
# down first. The front-end sets its own copy optimistically when it
# forwards the wake, so sample only after several coordinator
# publishes (every 100ms while stats change) have overwritten it.
# Rank 0 reports the wave only after its first step, which can outlast
# the 2s on a cold device, so wait for the report rather than sample it.
await asyncio.sleep(2)
assert not long_request.done(), "the warmup request was too short to sample"
assert engine.engine_core.dp_engines_running(), (
assert await _poll_flag(engine, True, timeout=30), (
"the coordinator does not report the engines as running while they are"
)
+270 -46
View File
@@ -4,17 +4,28 @@ import asyncio
import os
import threading
import time
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import AsyncExitStack
from types import SimpleNamespace
from typing import TYPE_CHECKING, cast
import msgspec
import openai # use the official client for correctness check
import pytest
import pytest_asyncio
import requests
from tests.utils import RemoteOpenAIServer
from vllm.distributed.elastic_ep.external_elastic_ep import (
ExternalElasticEPScaleCoordinator,
)
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.v1.engine.core_client import DPAsyncMPClient
MODEL_NAME = os.getenv("MODEL_NAME", "ibm-research/PowerMoE-3b")
ELASTIC_EP_MODEL_NAME = "deepseek-ai/DeepSeek-V2-Lite-Chat"
# Number of data parallel ranks for external LB testing
DP_SIZE = int(os.getenv("DP_SIZE", "2"))
@@ -22,6 +33,111 @@ DP_SIZE = int(os.getenv("DP_SIZE", "2"))
TP_SIZE = int(os.getenv("TP_SIZE", "1"))
class DictStore:
def __init__(self, values: dict[str, bytes]):
self.values = values
def check(self, keys: list[str]) -> bool:
return all(key in self.values for key in keys)
def get(self, key: str) -> bytes:
return self.values[key]
def set(self, key: str, value: bytes) -> None:
self.values[key] = value
def test_external_elastic_ep_visible_epoch_has_bootstrap_and_phase(monkeypatch):
client = SimpleNamespace(
vllm_config=SimpleNamespace(
parallel_config=SimpleNamespace(
data_parallel_master_port=12345,
_data_parallel_master_port_list=[12346, 12347],
)
)
)
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
control_store, reconfig_store = DictStore({}), DictStore({})
coordinator.reconfig_store_ref = reconfig_store
monkeypatch.setattr(
coordinator, "_setup_reconfig_bootstrap", lambda: ("127.0.0.1", 12348)
)
epoch, bootstrap = coordinator._prepare_reconfig_bootstrap(control_store, 2, 3)
for store in (control_store, reconfig_store):
# DictStore preserves insertion order: metadata must precede the epoch.
assert list(store.values)[-1] == coordinator.key("current_epoch")
assert store.get(coordinator.key(epoch, "bootstrap")) == msgspec.msgpack.encode(
bootstrap
)
assert store.get(coordinator.key(epoch, "phase")) == b"preparing"
assert store.get(coordinator.key("current_epoch")) == epoch.encode()
monkeypatch.setattr(
coordinator, "_get_reconfig_store", lambda store=store: store
)
assert coordinator.get_status()["requested_data_parallel_size"] == 3
def test_external_elastic_ep_calculates_target_expert_redundancy():
client = SimpleNamespace(
vllm_config=SimpleNamespace(
model_config=SimpleNamespace(get_num_experts=lambda: 128),
parallel_config=SimpleNamespace(
eplb_config=SimpleNamespace(num_redundant_experts=0)
),
)
)
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
assert coordinator._calculate_num_redundant_experts(2, 3) == 64
@pytest.mark.asyncio
@pytest.mark.parametrize("dp_rank", range(4))
async def test_validate_expert_capacity_on_every_rank(dp_rank, monkeypatch):
"""Validate expert capacity on every rank before waiting for an epoch."""
client = SimpleNamespace(
vllm_config=SimpleNamespace(
model_config=SimpleNamespace(get_num_experts=lambda: 128),
parallel_config=SimpleNamespace(
data_parallel_rank=dp_rank,
elastic_ep_max_dp_size=4,
data_parallel_master_ip="127.0.0.1",
_coord_store_port=12345,
eplb_config=SimpleNamespace(num_redundant_experts=0),
),
)
)
coordinator = ExternalElasticEPScaleCoordinator(cast("DPAsyncMPClient", client))
store = DictStore({})
monkeypatch.setattr(
"vllm.distributed.utils.get_cached_tcp_store_client", lambda *args: store
)
with pytest.raises(ValueError, match="cannot retain all model experts"):
await asyncio.wait_for(coordinator.prepare(4, 2), timeout=1)
assert not store.values
assert coordinator.prepared_scale is None
@pytest.mark.asyncio
async def test_external_elastic_ep_late_rank_observes_epoch_error():
coordinator = ExternalElasticEPScaleCoordinator(
cast("DPAsyncMPClient", SimpleNamespace())
)
epoch = "failed-epoch"
store = DictStore(
{
coordinator.key("current_epoch"): epoch.encode(),
coordinator.key(epoch, "error"): b"scale failed",
}
)
with pytest.raises(RuntimeError, match="scale failed"):
await coordinator._wait_for_bootstrap(store, requested_new_dp_size=3)
class ExternalLBServerManager:
"""Manages data parallel vLLM server instances for external
load balancer testing."""
@@ -42,57 +158,69 @@ class ExternalLBServerManager:
self.servers: list[tuple[RemoteOpenAIServer, list[str]]] = []
self.server_threads: list[threading.Thread] = []
def start_rank(
self,
rank: int,
dp_size: int | None = None,
elastic_ep_scale_up_launch: bool = False,
) -> tuple[RemoteOpenAIServer, list[str]]:
"""Start one externally managed DP rank."""
dp_size = self.dp_size if dp_size is None else dp_size
server_args = self.base_server_args.copy()
server_args.extend(
[
"--data-parallel-size",
str(dp_size),
"--data-parallel-rank",
str(rank),
"--data-parallel-size-local",
"1",
"--tensor-parallel-size",
str(self.tp_size),
"--port",
str(8000 + rank),
"--api-server-count",
str(self.api_server_count),
]
)
env_dict = {
"VLLM_SERVER_DEV_MODE": "1",
current_platform.device_control_env_var: ",".join(
str(current_platform.device_id_to_physical_device_id(i))
for i in range(rank * self.tp_size, (rank + 1) * self.tp_size)
),
}
if elastic_ep_scale_up_launch:
env_dict["VLLM_ELASTIC_EP_SCALE_UP_LAUNCH"] = "1"
server = RemoteOpenAIServer(
self.model_name,
server_args,
auto_port=False,
env_dict=env_dict,
)
print(
f"Server rank {rank} started successfully with "
f"{self.api_server_count} API servers"
)
server_and_args = (server, server_args)
self.servers.append(server_and_args)
return server_and_args
def __enter__(self) -> list[tuple[RemoteOpenAIServer, list[str]]]:
"""Start all server instances for external LB mode."""
"""Start all initial server instances for external LB mode."""
start_errors: list[Exception] = []
for rank in range(self.dp_size):
# Create server args for this specific rank
server_args = self.base_server_args.copy()
# Add external LB specific arguments
server_args.extend(
[
"--data-parallel-size",
str(self.dp_size),
"--data-parallel-rank",
str(rank),
"--data-parallel-size-local",
"1",
"--tensor-parallel-size",
str(self.tp_size),
"--port",
str(8000 + rank), # Different port for each rank
"--api-server-count",
str(self.api_server_count),
]
)
# Use a thread to start each server to allow parallel initialization
def start_server(r: int, sargs: list[str]):
def start_server(r: int):
try:
# Start the server
server = RemoteOpenAIServer(
self.model_name,
sargs,
auto_port=False,
env_dict={
"VLLM_SERVER_DEV_MODE": "1",
current_platform.device_control_env_var: ",".join(
str(current_platform.device_id_to_physical_device_id(i))
for i in range(r * TP_SIZE, (r + 1) * TP_SIZE)
),
},
)
server.__enter__()
print(
f"Server rank {r} started successfully with "
f"{self.api_server_count} API servers"
)
self.servers.append((server, sargs))
self.start_rank(r)
except Exception as e:
print(f"Failed to start server rank {r}: {e}")
raise
start_errors.append(e)
thread = threading.Thread(target=start_server, args=(rank, server_args))
thread = threading.Thread(target=start_server, args=(rank,))
thread.start()
self.server_threads.append(thread)
@@ -101,6 +229,10 @@ class ExternalLBServerManager:
for thread in self.server_threads:
thread.join()
if start_errors:
start_error = start_errors[0]
raise RuntimeError("Failed to start external LB servers") from start_error
# Give servers additional time to fully initialize and coordinate
time.sleep(2)
@@ -184,12 +316,104 @@ def _assert_expert_parallel_config(
assert parallel_config["enable_expert_parallel"] is expected
def _send_scale_command(
server: RemoteOpenAIServer, new_dp_size: int
) -> requests.Response:
return requests.post(
server.url_for("scale_elastic_ep"),
json={"new_data_parallel_size": new_dp_size},
timeout=600,
)
def _wait_for_scale_requests_to_start(
scale_requests: list[Future[requests.Response]],
timeout: float = 30,
) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
for request in scale_requests:
if request.done():
response = request.result()
pytest.fail(
"Scale request finished before the new rank was launched: "
f"{response.status_code} {response.text}"
)
if all(request.running() for request in scale_requests):
return
time.sleep(0.1)
pytest.fail("Timed out waiting for all scale requests to start")
@pytest.mark.distributed(num_gpus=3)
@pytest.mark.skipif(
TP_SIZE != 1 or current_platform.device_count() < 3,
reason="External Elastic EP scale-up requires three GPUs with TP_SIZE=1",
)
@pytest.mark.skipif(
os.getenv("VLLM_USE_RUST_FRONTEND") == "1",
reason="Elastic EP scaling is not supported by the Rust frontend",
)
def test_external_lb_elastic_ep_scale_up(default_server_args) -> None:
server_args = [
*default_server_args,
"--enable-expert-parallel",
"--all2all-backend",
"allgather_reducescatter",
"--attention-backend",
"TRITON_MLA",
"--enable-elastic-ep",
"--elastic-ep-max-dp-size",
"3",
"--enable-eplb",
"--eplb-config",
'{"communicator":"pynccl","num_redundant_experts":0,"use_async":false}',
]
server_manager = ExternalLBServerManager(
ELASTIC_EP_MODEL_NAME,
dp_size=2,
api_server_count=1,
base_server_args=server_args,
)
with server_manager:
existing_servers = [server for server, _ in server_manager.servers]
with ThreadPoolExecutor(max_workers=2) as executor:
scale_requests = [
executor.submit(_send_scale_command, server, 3)
for server in existing_servers
]
_wait_for_scale_requests_to_start(scale_requests)
server_manager.start_rank(
rank=2,
dp_size=3,
elastic_ep_scale_up_launch=True,
)
for request in scale_requests:
response = request.result(timeout=600)
assert response.status_code == 200, response.text
for server, _ in server_manager.servers:
parallel_config = _get_parallel_config(server)
assert parallel_config["data_parallel_size"] == 3
assert parallel_config["eplb_config"]["num_redundant_experts"] == 32
with server.get_client() as client:
completion = client.completions.create(
model=ELASTIC_EP_MODEL_NAME,
prompt="Hello, my name is",
max_tokens=5,
temperature=0.0,
)
assert completion.choices[0].finish_reason in ("length", "stop")
def test_external_lb_server_info(server_manager):
servers = server_manager.servers
api_server_count = server_manager.api_server_count
_assert_expert_parallel_config(servers)
for i, (server, _) in enumerate(servers):
print(f"Testing {i=}")
@@ -5,7 +5,13 @@ import pytest
from tests.utils import single_gpu_only
from vllm import SamplingParams
from vllm.platforms import current_platform
from ...utils import (
assert_request_outputs_match,
compute_acceptance_len,
get_test_prompts,
)
from .._correctness import check_mtp_correctness
@@ -37,3 +43,107 @@ def test_mtp_correctness(
expected_accuracy_threshold,
vllm_runner,
)
@single_gpu_only
def test_mtp_draft_lm_head_quantization(
monkeypatch: pytest.MonkeyPatch,
sampling_config: SamplingParams,
vllm_runner,
):
"""A quantized draft copy of the shared lm_head only changes drafting:
greedy outputs match stock MTP and acceptance stays within 10%."""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
prompts = get_test_prompts(mm_enabled=False)
results = {}
# NVFP4 needs a CUDA W4A16 kernel.
quants = (None, "fp8", "nvfp4") if current_platform.is_cuda() else (None, "fp8")
for quant in quants:
with vllm_runner(
"Qwen/Qwen3.5-0.8B-Base",
block_size=None,
enable_chunked_prefill=None,
max_model_len=2048,
limit_mm_per_prompt={"image": 0, "video": 0},
speculative_config={
"method": "mtp",
"num_speculative_tokens": 3,
"draft_lm_head_quantization": quant,
},
disable_log_stats=False,
) as runner:
outputs = runner.llm.chat(prompts, sampling_config)
results[quant] = (outputs, compute_acceptance_len(runner.llm.get_metrics()))
ref_outputs, ref_al = results.pop(None)
for quant, (outputs, al) in results.items():
assert_request_outputs_match(
ref_outputs,
outputs,
# Not bit-exact: a different acceptance length changes the target's
# verify batch shapes, which can flip a near-tied greedy token.
required_matches=int(0.8 * len(ref_outputs)) + 1,
context=f"draft_lm_head_quantization={quant}",
)
assert al >= 0.9 * ref_al, (
f"{quant}: acceptance length {al:.3f} vs {ref_al:.3f}"
)
@single_gpu_only
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="nvfp4 needs a CUDA W4A16 kernel"
)
def test_mtp_draft_lm_head_quantization_survives_sleep_and_reload(
monkeypatch: pytest.MonkeyPatch,
sampling_config: SamplingParams,
vllm_runner,
):
"""The quantized draft head is derived from the target head: it must come
back intact after a level 1 sleep and be re-derived after a target weight
reload, so outputs and acceptance match the run before."""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
prompts = get_test_prompts(mm_enabled=False)
with vllm_runner(
"Qwen/Qwen3.5-0.8B-Base",
block_size=None,
enable_chunked_prefill=None,
max_model_len=2048,
limit_mm_per_prompt={"image": 0, "video": 0},
speculative_config={
"method": "mtp",
"num_speculative_tokens": 3,
"draft_lm_head_quantization": "nvfp4",
},
enable_sleep_mode=True,
disable_log_stats=False,
) as runner:
llm = runner.llm
def run():
before = llm.get_metrics()
outputs = llm.chat(prompts, sampling_config)
return outputs, compute_acceptance_len(llm.get_metrics(), before)
ref_outputs, ref_al = run()
llm.sleep(level=1)
llm.wake_up()
outputs, al = run()
assert_request_outputs_match(
ref_outputs,
outputs,
required_matches=int(0.8 * len(ref_outputs)) + 1,
context="after level 1 sleep",
)
assert al == pytest.approx(ref_al, rel=0.03)
llm.collective_rpc("reload_weights")
outputs, al = run()
assert_request_outputs_match(
ref_outputs,
outputs,
required_matches=int(0.8 * len(ref_outputs)) + 1,
context="after weight reload",
)
assert al == pytest.approx(ref_al, rel=0.03)
@@ -141,7 +141,7 @@ def _make_addresses() -> EngineZmqAddresses:
False,
),
({"data_parallel_backend": "mp", "data_parallel_rank_local": 0}, False),
({"data_parallel_backend": "mp", "enable_elastic_ep": True}, False),
({"data_parallel_backend": "mp", "enable_elastic_ep": True}, True),
({"data_parallel_backend": "mp", "data_parallel_size": 1}, False),
],
ids=["mp", "ray", "non-master-node", "offline", "elastic-ep", "single-engine"],
@@ -150,8 +150,9 @@ def test_coordination_store_held_only_by_the_online_dp_master(
monkeypatch: pytest.MonkeyPatch, overrides: dict[str, Any], expect_store: bool
) -> None:
"""The DP master node holds a coordination store for both engine backends,
so engines there pick world-group ports at bind time. Every other
deployment keeps the pre-allocated ports and must not hold one."""
so engines there pick world-group ports at bind time. Elastic EP also
creates its coordination store before startup. The remaining cases
below keep the pre-allocated ports and must not hold a store."""
class FakeManager:
def __init__(self, **kwargs) -> None:
+52 -1
View File
@@ -142,6 +142,9 @@ def test_mp_client_uses_env_timeout(monkeypatch: pytest.MonkeyPatch):
def setsockopt(self, *_args, **_kwargs):
pass
def getsockopt_string(self, _option):
return "inproc://dummy"
monkeypatch.setattr(core_client_mod.zmq.Socket, "shadow", lambda *_: ShadowSocket())
monkeypatch.setattr(
core_client_mod, "make_zmq_socket", lambda *_, **__: DummySocket()
@@ -210,6 +213,7 @@ def _make_dplb_client(num_engines: int = 3, client_count: int = 1) -> DPLBAsyncM
client = object.__new__(DPLBAsyncMPClient)
client.client_count = client_count
client.reqs_in_flight = {}
client.streaming_reqs = set()
client.engine_inflight = Counter()
client.core_engines = [bytes([i, 0]) for i in range(num_engines)]
client.lb_engines = [[0, 0, 0.0] for _ in range(num_engines)]
@@ -407,11 +411,47 @@ def test_dplb_finished_requests_release_inflight():
assert req.request_id not in client.reqs_in_flight
@pytest.mark.parametrize("final_sent", [False, True])
def test_dplb_streaming_session_abort_goes_to_all_engines(final_sent: bool):
"""A streaming-input chunk in flight when its session finished may
re-create it on any engine, so an open session's abort reaches every
engine. Other aborts go only to the routed engine."""
client = _make_dplb_client(num_engines=3)
client.resources = SimpleNamespace(engine_dead=False)
sent: dict[bytes, list[str]] = {}
async def abort_requests(request_ids, engine):
sent[engine] = request_ids
client._abort_requests = abort_requests
chunk = make_request(SamplingParams(max_tokens=1))
chunk.resumable = True
client.get_core_engine_for_request(chunk)
if final_sent:
chunk.resumable = False
client.get_core_engine_for_request(chunk)
# The engine finished the session, dropping its routing entry.
outputs = EngineCoreOutputs(finished_requests={chunk.request_id})
asyncio.run(DPLBAsyncMPClient.process_engine_outputs(client, outputs))
client.reqs_in_flight["live"] = client.core_engines[1]
asyncio.run(client.abort_requests_async([chunk.request_id, "live"]))
if final_sent:
expected = {client.core_engines[1]: ["live"]}
else:
expected = {engine: [chunk.request_id] for engine in client.core_engines}
expected[client.core_engines[1]].append("live")
assert sent == expected
assert not client.streaming_reqs
@pytest.mark.parametrize(
("effective_size", "other_size"),
[(None, None), (4224, 4224), (4224, 1056), (4224, None)],
)
def test_apply_ready_response_syncs_block_size(effective_size, other_size):
def test_apply_ready_response_syncs_engine_metadata(effective_size, other_size):
import msgspec
client = object.__new__(MPClient)
@@ -419,8 +459,11 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
client.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
model_config=SimpleNamespace(max_model_len=8192),
parallel_config=SimpleNamespace(_coord_store_port=0),
)
client.stats_update_address = None
client.coordinator_input_address = None
client.coordinator_output_address = None
payload = msgspec.msgpack.encode(
EngineCoreReadyResponse(
@@ -441,6 +484,9 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
instance_id="test-instance",
supports_lora=False,
max_loras=0,
coord_store_port=1234,
coordinator_input_address="tcp://127.0.0.1:1235",
coordinator_output_address="tcp://127.0.0.1:1236",
)
)
fields = msgspec.msgpack.decode(payload)
@@ -461,6 +507,9 @@ def test_apply_ready_response_syncs_block_size(effective_size, other_size):
client._apply_ready_response(b"")
assert cache_config.effective_attention_block_size is None
assert client.vllm_config.parallel_config._coord_store_port == 1234
assert client.coordinator_input_address == "tcp://127.0.0.1:1235"
assert client.coordinator_output_address == "tcp://127.0.0.1:1236"
def test_apply_ready_response_syncs_mamba_block_size():
@@ -471,6 +520,7 @@ def test_apply_ready_response_syncs_mamba_block_size():
client.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
model_config=SimpleNamespace(max_model_len=8192),
parallel_config=SimpleNamespace(_coord_store_port=0),
)
client.stats_update_address = None
@@ -1541,6 +1591,7 @@ def test_apply_ready_response_retains_kv_event_sources():
client.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=0),
model_config=SimpleNamespace(max_model_len=8192),
parallel_config=SimpleNamespace(_coord_store_port=0),
)
def ready(rank: int, config: KVEventsConfig | None) -> bytes:
+132
View File
@@ -42,6 +42,7 @@ from vllm.v1.engine.output_processor import (
OutputProcessor,
RequestOutputCollector,
RequestState,
StreamingInputOutputCollector,
)
from vllm.v1.engine.parallel_sampling import ParentRequest
from vllm.v1.metrics.loggers import PrometheusStatLogger
@@ -1642,6 +1643,137 @@ async def test_cumulative_output_collector_n():
assert third[0].text == "c"
def _make_boundary_output(
token_id: int, finish_reason: str | None, finished: bool = False
) -> RequestOutput:
return RequestOutput(
request_id="my-request-id",
prompt=None,
prompt_token_ids=[1, 2, 3],
prompt_logprobs=None,
outputs=[
CompletionOutput(
index=0,
text="a",
token_ids=[token_id],
cumulative_logprob=None,
logprobs=None,
finish_reason=finish_reason,
)
],
finished=finished,
)
@pytest.mark.asyncio
async def test_streaming_input_collector_preserves_chunk_boundary():
"""A pending chunk-boundary marker (finish_reason on a non-finished
output) must not be erased by merging the next chunk's output into it."""
collector = StreamingInputOutputCollector(
RequestOutputKind.DELTA, request_id="my-request-id-int"
)
collector.put(_make_boundary_output(0, "stop"))
collector.put(_make_boundary_output(1, None))
collector.put(_make_boundary_output(2, None))
boundary = await collector.get()
assert boundary.outputs[0].finish_reason == "stop"
assert boundary.outputs[0].token_ids == [0]
following = await collector.get()
assert following.outputs[0].finish_reason is None
assert following.outputs[0].token_ids == [1, 2]
assert collector.output is None
assert not collector._backlog
assert not collector.ready.is_set()
def _make_streaming_input_request(
request_id: str, prompt_token_ids: list[int]
) -> EngineCoreRequest:
"""One chunk of a streaming-input (resumable) request."""
return EngineCoreRequest(
request_id=request_id,
prompt_token_ids=prompt_token_ids,
mm_features=None,
sampling_params=SamplingParams(
detokenize=False, output_kind=RequestOutputKind.DELTA
),
pooling_params=None,
arrival_time=0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
resumable=True,
external_req_id=f"{request_id}-ext",
)
def _add_streaming_request_with_queued_chunk(
processor: OutputProcessor, request_id: str
) -> RequestOutputCollector:
queue = RequestOutputCollector(RequestOutputKind.DELTA, request_id=request_id)
processor.add_request(
_make_streaming_input_request(request_id, [1, 2, 3]), None, queue=queue
)
processor.add_request(_make_streaming_input_request(request_id, [7, 8]), None)
return queue
def test_streaming_input_stop_is_chunk_boundary():
"""STOP is a per-chunk boundary: finished is forced False, the queued
chunk is applied, and the request state stays alive."""
processor = OutputProcessor(None, log_stats=False)
queue = _add_streaming_request_with_queued_chunk(processor, "stream-req")
processed = processor.process_outputs(
[
EngineCoreOutput(
request_id="stream-req",
new_token_ids=[4],
finish_reason=FinishReason.STOP,
)
]
)
assert processed.reqs_to_abort == []
assert processor.get_num_unfinished_requests() == 1
output = queue.get_nowait()
assert output is not None
assert output.finished is False
assert output.outputs[0].finish_reason == "stop"
req_state = processor.request_states["stream-req"]
assert req_state.prompt_token_ids is not None
assert req_state.prompt_token_ids[-2:] == [7, 8]
@pytest.mark.parametrize("finish_reason", [FinishReason.ABORT, FinishReason.ERROR])
def test_streaming_input_abort_error_is_terminal(finish_reason: FinishReason):
"""ABORT/ERROR is terminal, not a chunk boundary: finished stays True,
the state is freed, and an abort is routed back to the engine core."""
processor = OutputProcessor(None, log_stats=False)
queue = _add_streaming_request_with_queued_chunk(processor, "stream-req")
processed = processor.process_outputs(
[
EngineCoreOutput(
request_id="stream-req",
new_token_ids=[],
finish_reason=finish_reason,
)
]
)
assert processor.get_num_unfinished_requests() == 0
assert processed.reqs_to_abort == ["stream-req"]
output = queue.get_nowait()
assert output is not None
assert output.finished is True
assert output.outputs[0].finish_reason == str(finish_reason)
@pytest.mark.parametrize("runner", ["generate", "pooling"])
@pytest.mark.parametrize("abort_by", ["internal", "external"])
def test_abort_requests(runner: str, abort_by: str, dummy_test_vectors):
@@ -281,6 +281,7 @@ class _FinishedProcess:
def test_wait_for_engine_startup_reports_watched_process_exit():
"""Update the startup test argument so it reaches frontend failure handling."""
ctx = zmq.Context()
handshake_socket = ctx.socket(zmq.ROUTER)
recv, send = connection.Pipe(duplex=False)
@@ -190,11 +190,13 @@ def _apply_ft(server, instruction: str, params: dict | None = None) -> dict:
def _kill_worker_process(server) -> None:
"""SIGKILL only the worker proc, leaving EngineCore and API server alive."""
workers = [
p
for p in psutil.Process(server.proc.pid).children(recursive=True)
if "Worker" in " ".join(p.cmdline())
]
workers = []
for p in psutil.Process(server.proc.pid).children(recursive=True):
# Skip descendants that already exited: cmdline() raises ZombieProcess
# if unreaped, NoSuchProcess if reaped. A dead worker still fails below.
with contextlib.suppress(psutil.ZombieProcess, psutil.NoSuchProcess):
if "Worker" in " ".join(p.cmdline()):
workers.append(p)
assert len(workers) == 1, f"expected 1 worker proc, found: {workers}"
workers[0].kill()
@@ -73,6 +73,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheLayout,
KVQuantMode,
MLAAttentionSpec,
compute_layer_kv_cache_shape_bytes,
)
@@ -1551,6 +1552,95 @@ class TestNixlHandshake:
with pytest.raises(AssertionError):
worker.add_remote_agent(bad_meta, remote_tp_size=2)
@pytest.mark.parametrize(
("kv_quant_mode", "local_tp", "remote_tp", "num_kv_heads", "rejected"),
[
(KVQuantMode.NVFP4, 2, 1, 8, True),
(KVQuantMode.NVFP4, 1, 2, 8, True),
(KVQuantMode.NVFP4, 2, 2, 8, False),
# Both sides hold one replicated head per rank: no head split.
(KVQuantMode.NVFP4, 16, 8, 8, False),
(KVQuantMode.NONE, 2, 1, 8, False),
],
)
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FakeNixlWrapper,
)
def test_handshake_rejects_nvfp4_kv_head_split(
self,
default_vllm_config,
dist_init,
kv_quant_mode,
local_tp,
remote_tp,
num_kv_heads,
rejected,
):
"""NVFP4 pages keep all heads' data before all heads' scales, so
heterogeneous-TP head splitting would silently corrupt the KV.
"""
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(block_size=block_size)
kv_cache_config = KVCacheConfig(
num_blocks=1,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=max(1, num_kv_heads // local_tp),
head_size=16,
dtype=torch.uint8,
kv_quant_mode=kv_quant_mode,
),
)
],
)
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501
return_value=local_tp,
):
worker = FakeNixlConnectorWorker(
vllm_config,
"engine",
hand_shake_latency=0,
kv_cache_config=kv_cache_config,
)
worker.transfer_topo.total_num_kv_heads = num_kv_heads
worker.transfer_topo.local_physical_heads = max(1, num_kv_heads // local_tp)
local_len = 4096 * worker.block_size
remote_len = (
local_len
* max(1, num_kv_heads // remote_tp)
// worker.transfer_topo.local_physical_heads
)
worker.block_len_per_layer = [local_len]
worker.num_blocks = 1
worker.dst_num_blocks[worker.engine_id] = worker.num_blocks
meta = NixlAgentMetadata(
engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
agent_metadata=FakeNixlWrapper.AGENT_METADATA,
kv_caches_base_addr=[0],
device_id=0,
num_blocks=1,
block_lens=[remote_len],
block_strides=[remote_len],
kv_cache_layout=worker.kv_cache_layout,
block_size=worker.block_size,
ssm_sizes=(0, 0),
attn_backend_name=worker.backend_name,
physical_blocks_per_logical_kv_block=1,
)
if rejected:
with pytest.raises(RuntimeError, match="NVFP4"):
worker.add_remote_agent(meta, remote_tp_size=remote_tp)
else:
worker.add_remote_agent(meta, remote_tp_size=remote_tp)
# NOTE: resource cleanup in mp backend is a bit finicky, so the order in which
# we put here is important. First run ray, it will clean up the resources, then
@@ -653,6 +653,41 @@ def test_madvise_unexpected_oserror_propagates(iid, monkeypatch):
assert exc_info.value.errno == errno.EIO
def test_segmented_region_maps_files_back_to_back(iid, monkeypatch):
"""Rows split over three shm files mapped back to back: each worker sees
the other's writes in every file, and the creator removes every file."""
monkeypatch.setattr(region_module, "_MIN_SEGMENT_BYTES", PAGE_SIZE)
regions = [
SharedOffloadRegion(
engine_id=iid,
num_chunks=5,
rank=rank,
kv_bytes_per_chunk=2 * PAGE_SIZE,
cpu_page_size=PAGE_SIZE,
num_segments=3,
)
for rank in range(2)
]
segments = regions[0]._segments
try:
assert [length // PAGE_SIZE for _, _, length in segments] == [4, 4, 2]
for rank, region in enumerate(regions):
ranges = list(region.iter_worker_slot_ranges())
first_row = rank * 5 // 2
assert ranges[0] == ((first_row * 2 + rank) * PAGE_SIZE, PAGE_SIZE)
assert sorted(ranges) == [
((row * 2 + rank) * PAGE_SIZE, PAGE_SIZE) for row in range(5)
]
region.create_next_worker_view(PAGE_SIZE).fill_(rank + 1)
rows = regions[0].base_tensor.view(5, 2, PAGE_SIZE)
assert (rows[:, 0] == 1).all() and (rows[:, 1] == 2).all()
del rows
finally:
for region in regions:
region.cleanup()
assert not any(os.path.exists(path) for path, _, _ in segments)
# ---------------------------------------------------------------------------
# Multi-worker race — concurrent construction
# ---------------------------------------------------------------------------
@@ -899,6 +934,7 @@ def _unmapped_registration_region(num_chunks, slot_size, num_workers):
region.cpu_page_size = slot_size
region._row_stride = num_workers * slot_size
region.total_size_bytes = region.num_chunks * region._row_stride
region._segments = [("unused", 0, region.total_size_bytes)]
region._base = SimpleNamespace(data_ptr=lambda: 0x100000)
region.pinned_addresses = []
region.is_pinned = False
+42
View File
@@ -15,6 +15,8 @@ from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import
)
from vllm.model_executor.models.qwen3_dspark import DSparkMarkovHead
from vllm.platforms import current_platform
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.v1.worker.gpu.spec_decode.dspark.speculator import DSparkSpeculator
def _markov_head(weight: torch.Tensor) -> DSparkMarkovHead:
@@ -70,6 +72,46 @@ def test_gathered_markov_bias_matches_dense_at_full_vocab():
torch.testing.assert_close(result, expected)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("use_fp64", [False, True])
def test_dspark_revisited_position_uses_step_seed(use_fp64):
"""A cached draft at a revisited position must use its current step."""
device = torch.device("cuda")
batch = 4096
spec = DSparkSpeculator.__new__(DSparkSpeculator)
spec.acceptance_estimator = None
spec._d2t_scatter_index = None
spec.draft_watermarker = None
spec.use_fp64_gumbel = use_fp64
spec.temperature = torch.ones(batch, device=device)
spec.seeds = torch.arange(batch, dtype=torch.int64, device=device)
spec._step_cols = torch.arange(2, dtype=torch.int32, device=device)
spec.draft_logits = torch.empty(batch, 2, 2, device=device)
logits = torch.tensor([0.7, 0.3], device=device).log().expand(batch, -1)
mapping = torch.arange(batch, device=device)
positions = torch.full((batch,), 22, dtype=torch.int64, device=device)
first = spec._sample_logits(logits, mapping, positions, 1)
repeated = spec._sample_logits(logits, mapping, positions, 1)
next_round = spec._sample_logits(logits, mapping, positions, 0)
torch.testing.assert_close(first, repeated)
expected = gumbel_sample(
logits,
mapping,
spec.temperature,
spec.seeds,
positions - 1,
apply_temperature=True,
is_drafting=True,
logits_cache=spec.draft_logits,
logits_cache_col=spec._step_cols[0],
use_fp64=use_fp64,
)
torch.testing.assert_close(next_round, expected)
mismatch_rate = (first != next_round).float().mean().item()
assert 0.35 < mismatch_rate < 0.5
@pytest.mark.parametrize("cuda_alike_platform", [False, True])
def test_gathered_markov_bias_dequantizes_selected_w4a16_rows(
monkeypatch, cuda_alike_platform
+55
View File
@@ -22,6 +22,7 @@ from vllm.model_executor.layers import logits_processor
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.models.lilicorr import LiLiCorrHead
from vllm.transformers_utils.configs.eagle import EAGLEConfig
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import CandidateSampler
from vllm.v1.worker.gpu.spec_decode.lilicorr.speculator import LiLiCorrSpeculator
@@ -500,6 +501,60 @@ def test_candidate_walk_preserves_conditional_scores_and_padding(probabilistic):
assert cache[1].isneginf().all()
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("use_fp64", [False, True])
def test_candidate_walk_revisited_position_matches_step_seed(use_fp64):
"""The candidate walk must use the same step key as ordinary draft sampling."""
device = torch.device("cuda")
batch, steps, top_k = 4096, 2, 2
sampler = CandidateSampler(batch, steps, top_k, device)
candidates = torch.arange(top_k, device=device).expand(batch, steps, top_k)
logits = torch.tensor([0.7, 0.3], device=device).log()
scores = logits.expand(batch, steps, top_k, top_k)
mapping = torch.arange(batch, device=device)
temperature = torch.ones(batch, device=device)
seeds = mapping.to(torch.int64)
positions = torch.arange(21, 21 + steps, device=device).repeat(batch)
tokens = torch.empty(batch, steps, dtype=torch.int64, device=device)
cache = torch.empty(batch, steps, top_k, device=device)
first_round = None
for start in (21, 22):
positions.view(batch, steps).copy_(
torch.arange(start, start + steps, device=device)
)
sampler.sample(
candidates,
scores,
batch,
positions,
mapping.repeat_interleave(steps),
temperature,
seeds,
tokens,
cache,
use_fp64,
)
for step in range(steps):
expected = gumbel_sample(
logits.expand(batch, -1),
mapping,
temperature,
seeds,
positions.view(batch, steps)[:, step] - 1,
apply_temperature=True,
is_drafting=True,
logits_cache=cache,
logits_cache_col=torch.tensor(step, device=device),
use_fp64=use_fp64,
)
torch.testing.assert_close(tokens[:, step], expected)
if first_round is None:
first_round = tokens.clone()
else:
mismatch_rate = (first_round[:, 1] != tokens[:, 0]).float().mean().item()
assert 0.35 < mismatch_rate < 0.5
@pytest.mark.parametrize(
("convolution", "mismatch", "quantized"),
[
@@ -114,6 +114,10 @@ class _EngineReasonerStub(ParserEngineReasoningAdapter):
def reasoning_end_token_ids(self):
return self._end_token_ids
@property
def implicit_reasoning_end_token_ids(self):
return frozenset()
def find_reasoning_end(self, token_ids):
self.windows.append(list(token_ids))
for offset, token in enumerate(token_ids):
@@ -557,6 +557,79 @@ def test_gumbel_drafted_rejection_sample_is_unbiased(num_speculative_steps: int)
)
@pytest.mark.parametrize("use_block_verification", [False, True])
@pytest.mark.parametrize("probabilistic_draft", [False, True])
def test_multiround_rejection_sample_is_unbiased(
use_block_verification: bool, probabilistic_draft: bool
):
"""Uncommitted positions must receive fresh noise on later verification rounds."""
torch.manual_seed(42)
num_trials, num_tokens, k = 32768, 6, 2
device = "cuda"
target_probs = torch.tensor(
[0.3, 0.7] if probabilistic_draft else [0.5, 0.5], device=device
)
draft_logits = torch.tensor(
[0.7, 0.3] if probabilistic_draft else [1.0, 0.0], device=device
).log()
inputs = _build_rejection_sample_inputs(
target_probs.log(), draft_logits, k, 1.0, num_trials
)
seeds = inputs["seed"]
output = torch.full((num_trials, num_tokens + k), -1, device=device)
output[:, 0] = torch.multinomial(target_probs, num_trials, replacement=True)
lengths = torch.ones(num_trials, dtype=torch.int64, device=device)
accepted = 0
while (lengths < num_tokens).any():
active = (lengths < num_tokens).nonzero().flatten()
n = active.numel()
starts = lengths[active]
local_positions = torch.arange(k + 1, device=device).repeat(n)
positions = (starts[:, None] + torch.arange(k + 1, device=device)).flatten()
draft_sampled = torch.zeros((n, k + 1), dtype=torch.int64, device=device)
draft_sampled[:, 0] = output[active, starts - 1]
logits_cache = inputs["draft_logits"] if probabilistic_draft else None
if probabilistic_draft:
for step in range(k):
draft_sampled[:, step + 1] = gumbel_sample(
draft_logits.expand(n, -1),
active,
inputs["temperature"],
seeds,
starts + step,
apply_temperature=True,
is_drafting=True,
logits_cache=logits_cache,
logits_cache_col=torch.tensor(step, device=device),
)
sampled, counts = rejection_sample(
target_logits=target_probs.log().expand(n * (k + 1), -1).contiguous(),
draft_logits=logits_cache,
draft_sampled=draft_sampled.flatten(),
cu_num_logits=torch.arange(n + 1, dtype=torch.int32, device=device)
* (k + 1),
pos=positions,
idx_mapping=active,
expanded_idx_mapping=active.repeat_interleave(k + 1),
expanded_local_pos=local_positions,
temperature=inputs["temperature"],
seed=seeds,
num_speculative_steps=k,
use_block_verification=use_block_verification,
)
for column in range(k + 1):
valid = counts > column
output[active[valid], starts[valid] + column] = sampled[valid, column]
lengths[active] += counts
accepted += (counts - 1).sum().item()
assert accepted > num_trials, "The test must exercise accepted draft tokens."
for position in range(num_tokens):
_assert_distribution_match(
output[:, position], target_probs, device, label=f"position {position}"
)
@pytest.mark.parametrize("num_speculative_steps", [1, 3])
def test_greedy_rejection_sample(num_speculative_steps: int):
"""Verify that greedy (temperature=0) always outputs the target argmax
@@ -3,6 +3,7 @@
import asyncio
from collections.abc import AsyncGenerator
from unittest import mock
from unittest.mock import AsyncMock, MagicMock
import pytest
@@ -198,3 +199,52 @@ async def test_empty_input_stream_finishes_without_engine_request():
assert queue.get_nowait() is STREAM_FINISHED
llm._add_request.assert_not_awaited()
def _make_streaming_llm(num_chunks: int, has_request: bool = True):
llm = AsyncLLM.__new__(AsyncLLM)
llm._validate_streaming_input_sampling_params = MagicMock()
llm.get_supported_tasks = AsyncMock(return_value=("generate",))
llm._run_output_handler = MagicMock()
llm._add_request = AsyncMock()
llm.model_config = MagicMock()
llm.output_processor = MagicMock()
llm.output_processor.has_request.return_value = has_request
# The first processed request is the final (end-of-stream) request.
reqs = [MagicMock(prompt_embeds=None) for _ in range(num_chunks + 1)]
reqs[0].request_id = "r1-int"
llm.input_processor = MagicMock()
llm.input_processor.process_inputs.side_effect = reqs
return llm, reqs
async def _stream_chunks(llm: AsyncLLM, num_chunks: int) -> None:
async def chunks():
for i in range(num_chunks):
yield StreamingInput(prompt=f"frame-{i}")
with mock.patch(
"vllm.v1.engine.async_llm.extract_prompt_components",
return_value=("", None, None),
):
queue = await llm._add_streaming_input_request(
"r1", chunks(), SamplingParams(max_tokens=4)
)
task = queue._input_stream_task
assert task is not None
await task
@pytest.mark.asyncio
@pytest.mark.parametrize("session_live", [True, False])
async def test_handle_inputs_stops_after_session_finished(session_live: bool):
"""Once the session's request state is gone (aborted or errored), no
further chunks or final request are sent: they would recreate a request
state that is never cleaned up."""
llm, reqs = _make_streaming_llm(num_chunks=2, has_request=session_live)
await _stream_chunks(llm, num_chunks=2)
sent = [call.args[0] for call in llm._add_request.await_args_list]
# Both chunks then the final request, or only the first chunk.
assert sent == ([reqs[1], reqs[2], reqs[0]] if session_live else [reqs[1]])
@@ -5,9 +5,11 @@ import unittest
from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
import vllm.v1.core.kv_cache_utils as kv_cache_utils
from tests.v1.core.utils import create_scheduler as create_spec_scheduler
from vllm.config import DeviceConfig, VllmConfig
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
@@ -28,7 +30,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
)
from vllm.v1.outputs import LogprobsLists, ModelRunnerOutput
from vllm.v1.outputs import DraftTokenIds, LogprobsLists, ModelRunnerOutput
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
from vllm.v1.structured_output import StructuredOutputManager
@@ -676,3 +678,41 @@ class TestStreamingScheduler(unittest.TestCase):
cached_state_cycle1["prompt_token_ids"]
is not cached_state_cycle3["prompt_token_ids"]
), "Cached states from different cycles should be independent objects."
def _model_output(request_id: str, token_ids: list[int]) -> ModelRunnerOutput:
return ModelRunnerOutput(
req_ids=[request_id],
req_id_to_index={request_id: 0},
sampled_token_ids=[token_ids],
logprobs=None,
prompt_logprobs_dict={request_id: None},
pooler_output=[],
)
@pytest.mark.parametrize("async_scheduling", [False, True])
def test_next_chunk_fully_computed_after_stop_on_accepted_draft(async_scheduling):
"""A stop on an accepted draft trims the step's later tokens. The next
chunk must still be computed in full rather than landing on positions
whose KV belongs to the trimmed drafts."""
scheduler = create_spec_scheduler(
num_speculative_tokens=2,
async_scheduling=async_scheduling,
speculative_method="ngram_gpu" if async_scheduling else None,
)
session = DummyRequest("session", prompt_token_ids=[1, 2, 3], max_tokens=2)
scheduler.add_request(session)
scheduler.update_from_output(scheduler.schedule(), _model_output("session", [10]))
scheduler.update_draft_token_ids(DraftTokenIds(["session"], [[11, 12]]))
# Both drafts are accepted, but max_tokens stops at 11.
scheduler.update_from_output(
scheduler.schedule(), _model_output("session", [11, 12, 13])
)
assert session.status == RequestStatus.WAITING_FOR_STREAMING_REQ
scheduler.add_request(DummyRequest("session", prompt_token_ids=[7, 8]))
output = scheduler.schedule()
assert list(session.prompt_token_ids) == [1, 2, 3, 10, 7, 8]
assert output.num_scheduled_tokens["session"] == 2
@@ -6,6 +6,7 @@ from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass
from threading import Event
import numpy as np
import pytest
from transformers import AutoTokenizer
@@ -72,20 +73,26 @@ class MockReasoner:
class MockEngineReasoner(ParserEngineReasoningAdapter):
"""Ends reasoning implicitly on ``marker``, which is also content."""
"""Ends reasoning implicitly on ``marker``, which is also content, and
explicitly on ``end_marker`` when one is given."""
def __init__(self, tokenizer, marker: int):
def __init__(self, tokenizer, marker: int, end_marker: int | None = None):
self.marker = marker
self.end_marker = end_marker
@property
def reasoning_end_token_ids(self):
return frozenset({self.marker, self.end_marker} - {None})
@property
def implicit_reasoning_end_token_ids(self):
return frozenset({self.marker})
def find_reasoning_end(self, token_ids):
ids = list(token_ids)
if self.marker in ids:
return ReasoningEnd(ids.index(self.marker), True)
return ReasoningEnd(len(ids), False)
for offset, token in enumerate(token_ids):
if token in self.reasoning_end_token_ids:
return ReasoningEnd(offset, token == self.marker)
return ReasoningEnd(len(token_ids), False)
def _single_token(tokenizer, text: str) -> int:
@@ -122,6 +129,7 @@ def _build_harness(
enable_in_reasoning: bool = False,
reasoning_parser_kwargs: dict | None = None,
structured_outputs: StructuredOutputsParams | None = None,
num_spec_tokens: int = NUM_SPEC_TOKENS,
) -> tuple[StructuredOutputManager, Request]:
vllm_config = VllmConfig(
model_config=ModelConfig(tokenizer=TOKENIZER),
@@ -132,8 +140,10 @@ def _build_harness(
),
speculative_config=SpeculativeConfig(
model="[ngram]",
num_speculative_tokens=NUM_SPEC_TOKENS,
),
num_speculative_tokens=num_spec_tokens,
)
if num_spec_tokens
else None,
)
manager = StructuredOutputManager(vllm_config)
if use_reasoner:
@@ -171,8 +181,20 @@ def _build_harness(
return manager, request
def _row_pattern(bitmask) -> str:
return "".join("U" if (row == -1).all() else "C" for row in bitmask)
def _row_pattern(bitmask, masked_ids=()) -> str:
"""U: unconstrained; R: all but `masked_ids` allowed; C: grammar rows."""
reasoning_row = np.full(bitmask.shape[1], -1, dtype=np.int32)
for token_id in masked_ids:
reasoning_row.view(np.uint32)[token_id >> 5] &= ~np.uint32(1 << (token_id & 31))
def kind(row) -> str:
if (row == -1).all():
return "U"
if masked_ids and (row == reasoning_row).all():
return "R"
return "C"
return "".join(kind(row) for row in bitmask)
def _run_real_flow(
@@ -183,6 +205,7 @@ def _run_real_flow(
expected_row_pattern: str,
expected_reasoning: bool | None,
expect_terminated: bool,
masked_ids: list[int] | None = None,
) -> None:
assert len(expected_validated) <= len(raw_drafts)
@@ -197,7 +220,7 @@ def _run_real_flow(
)
assert bitmask is not None
assert bitmask.shape[0] == len(raw_drafts) + 1
assert _row_pattern(bitmask) == expected_row_pattern
assert _row_pattern(bitmask, masked_ids or ()) == expected_row_pattern
# grammar_bitmask() must rollback any speculative state it advanced.
assert manager.validate_tokens(request, list(raw_drafts)) == expected_validated
@@ -375,7 +398,8 @@ FLOW_CASES = [
FlowCase(
raw_drafts=(" ", IMPLICIT_THINK_END, "{", "}"),
expected_validated=(" ",),
expected_row_pattern="UUUUU",
# The grammar cannot start with the marker, so reasoning rows mask it.
expected_row_pattern="RRUUU",
expected_reasoning=False,
expect_terminated=False,
engine_reasoner=True,
@@ -421,9 +445,93 @@ def test_real_flow(
expected_row_pattern=case.expected_row_pattern,
expected_reasoning=case.expected_reasoning,
expect_terminated=case.expect_terminated,
masked_ids=[reasoner_kwargs["marker"]] if case.engine_reasoner else None,
)
@pytest.mark.parametrize("backend", BACKENDS)
@pytest.mark.parametrize(
("structured_outputs", "raw_drafts", "expected_row_pattern"),
[
# JSON cannot start with the marker: reasoning rows mask it and keep
# every other token, including the explicit end, allowed.
pytest.param(
StructuredOutputsParams(json=JSON_SCHEMA), (" ", " "), "RRR", id="json"
),
# The grammar can start with the marker: it stays allowed and, once
# sampled, is the first token the grammar consumes.
pytest.param(
StructuredOutputsParams(regex="z[0-9]+"),
(" ", IMPLICIT_THINK_END, "1"),
"UUCC",
id="regex_starts_with_marker",
),
],
)
def test_reasoning_rows_mask_grammar_invalid_implicit_end(
tokenizer,
backend: StructuredOutputsBackend,
structured_outputs: StructuredOutputsParams,
raw_drafts: tuple[str, ...],
expected_row_pattern: str,
):
marker = _single_token(tokenizer, IMPLICIT_THINK_END)
manager, request = _build_harness(
tokenizer,
backend,
reasoning_ended=False,
reasoning_parser_kwargs={
"marker": marker,
"end_marker": _single_token(tokenizer, THINK_END),
},
structured_outputs=structured_outputs,
)
manager.reasoner_cls = MockEngineReasoner
drafts = _to_token_ids(tokenizer, raw_drafts)
assert manager.validate_tokens(request, drafts) == drafts
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={request.request_id: drafts},
)
assert bitmask is not None
assert _row_pattern(bitmask, [marker]) == expected_row_pattern
@pytest.mark.parametrize("backend", BACKENDS)
def test_parallel_fill_masks_grammar_invalid_implicit_end(
tokenizer, backend: StructuredOutputsBackend
):
marker, end = _to_token_ids(tokenizer, (IMPLICIT_THINK_END, THINK_END))
manager, request = _build_harness(
tokenizer,
backend,
reasoning_ended=False,
reasoning_parser_kwargs={"marker": marker, "end_marker": end},
num_spec_tokens=0,
)
manager.reasoner_cls = MockEngineReasoner
def row_pattern() -> str:
bitmask = manager.grammar_bitmask(
requests={request.request_id: request},
structured_output_request_ids=[request.request_id],
scheduled_spec_decode_tokens={},
)
assert bitmask is not None
return _row_pattern(bitmask, [marker])
manager.fill_bitmask_parallel_threshold = 0
manager.fill_bitmask_parallel_batch_size = 16
with ThreadPoolExecutor(max_workers=1) as executor:
manager.executor_for_fillmask = executor
assert row_pattern() == "R"
request.append_output_token_ids([end])
assert manager.accept_tokens(request, [end])
assert row_pattern() == "C"
@pytest.mark.parametrize("backend", BACKENDS)
@pytest.mark.parametrize(
("use_reasoner", "reasoning_ended", "enable_in_reasoning"),
+47 -25
View File
@@ -9,12 +9,14 @@ never addressed by the logical view.
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
import vllm.v1.hisparse.binding as attn_utils_module
import vllm.v1.hisparse.runtime as hisparse_runtime_module
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
@@ -403,49 +405,63 @@ class _FakeSharedHostRegion:
self.cleanup_calls += 1
def test_profiling_cleanup_releases_tp_shared_region_once(monkeypatch):
"""TP-shared profiling pools must use region-aware chunk cleanup."""
region = _FakeSharedHostRegion()
def test_profiling_cleanup_keeps_host_pool_for_real_kv_cache(monkeypatch):
"""The real KV cache must reuse the profiling host pool, not pin it again."""
pinned = torch.empty(8, dtype=torch.int8)
allocate_pinned = MagicMock(return_value=(pinned, pinned))
monkeypatch.setattr(
hisparse_runtime_module, "allocate_pinned_host_pool", allocate_pinned
)
monkeypatch.setattr(
hisparse_runtime_module, "check_hisparse_host_memory", lambda size: None
)
vllm_config = SimpleNamespace(instance_id="profiling-reuse")
kv_cache_config = SimpleNamespace(hisparse_shared_host_pool=False)
pool = attn_utils_module.get_hisparse_host_pool(vllm_config, kv_cache_config)
backing = pool.allocate(8)
runtime = SimpleNamespace(
_host_cache=object(),
registered_host_pool=region.base_tensor,
hot_backing=object(),
shared_host_region=region,
_host_cache=backing, registered_host_pool=pinned, hot_backing=object()
)
forward_context = {
"layer": SimpleNamespace(
hisparse_cache=SimpleNamespace(runtime=runtime),
)
"layer": SimpleNamespace(hisparse_cache=SimpleNamespace(runtime=runtime))
}
released = []
def release_pinned_state(runtimes, pinned_host_pools, shared_host_region):
released.append((runtimes, pinned_host_pools, shared_host_region))
monkeypatch.setattr(
attn_utils_module,
"release_pinned_state",
release_pinned_state,
)
attn_utils_module.release_hisparse_profiling_cache(forward_context)
assert released == [([runtime], [], region)]
assert not hasattr(runtime, "_host_cache")
reused = attn_utils_module.get_hisparse_host_pool(vllm_config, kv_cache_config)
assert reused is pool
assert reused.allocate(8) is backing
allocate_pinned.assert_called_once_with(8)
hisparse_runtime_module.drop_hisparse_host_pool(vllm_config)
@pytest.mark.parametrize("failure_phase", ["allocation", "binding", "buffers"])
def test_init_hisparse_rolls_back_shared_region(monkeypatch, failure_phase):
"""A failure after mmap allocation must not leak the shared registration."""
@pytest.mark.parametrize("shared", [False, True])
def test_init_hisparse_rolls_back_host_pool(monkeypatch, failure_phase, shared):
"""Failed initialization must release shared or private registrations."""
region = _FakeSharedHostRegion()
registered = torch.empty(4096, dtype=torch.int8)
cudart = MagicMock()
cudart.cudaHostUnregister.return_value.value = 0
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
monkeypatch.setattr(torch.cuda, "cudart", lambda: cudart)
vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC
),
scheduler_config=SimpleNamespace(max_num_seqs=1),
instance_id="rollback",
)
pools = []
def allocate(*args):
args[-1].shared_region = region
pools.append(args[-1])
if shared:
args[-1].shared_region = region
else:
args[-1].registered = registered
args[-1].backing = registered[:8]
if failure_phase == "allocation":
raise RuntimeError("initialization failed")
return {}
@@ -472,7 +488,13 @@ def test_init_hisparse_rolls_back_shared_region(monkeypatch, failure_phase):
{},
SimpleNamespace(),
)
assert region.cleanup_calls == 1
assert region.cleanup_calls == int(shared)
if shared:
cudart.cudaHostUnregister.assert_not_called()
else:
cudart.cudaHostUnregister.assert_called_once_with(registered.data_ptr())
assert attn_utils_module.get_hisparse_host_pool(vllm_config, None) is not pools[0]
hisparse_runtime_module.drop_hisparse_host_pool(vllm_config)
def test_reshape_padded_kv_cache_strides_by_padded_page():
@@ -0,0 +1,70 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The CPU worker's automatic KV cache sizing must report the right cause.
A worker whose own footprint exceeds its reservation and a node that is short
of free memory need opposite remedies, so they get separate errors.
"""
from types import SimpleNamespace
import pytest
from vllm.platforms import current_platform
from vllm.utils.cpu_resource_utils import MemoryNodeInfo
# Importing the CPU worker patches torch and the GPU buffer utilities for the
# whole process (vllm/v1/worker/cpu/shm.py), so the import is deferred into
# the tests and they run only on the CPU platform.
pytestmark = pytest.mark.skipif(
not current_platform.is_cpu(), reason="CPU worker test; needs the CPU platform"
)
GiB = 1 << 30
def _worker(monkeypatch, *, reserved, rss, total=8 * GiB, available=4 * GiB):
import vllm.v1.worker.cpu_worker as cpu_worker_mod
from vllm.v1.worker.cpu_worker import CPUWorker
worker = CPUWorker.__new__(CPUWorker)
worker.requested_cpu_memory = reserved
worker.cache_config = SimpleNamespace(kv_cache_memory_bytes=None)
monkeypatch.setattr(CPUWorker, "_should_warm_up_model", lambda self: False)
monkeypatch.setattr(
cpu_worker_mod,
"get_allowed_cpu_list",
lambda: [SimpleNamespace(numa_node=0)],
)
monkeypatch.setattr(
cpu_worker_mod,
"get_memory_node_info",
lambda node: MemoryNodeInfo(total_memory=total, available_memory=available),
)
process = SimpleNamespace(memory_info=lambda: SimpleNamespace(rss=rss))
monkeypatch.setattr(cpu_worker_mod.psutil, "Process", lambda pid: process)
return worker
def test_footprint_above_reservation_asks_for_a_larger_reservation(monkeypatch):
# 0.76 GiB already in use against a 0.16 GiB reservation (2% of 8 GiB).
worker = _worker(monkeypatch, reserved=int(0.16 * GiB), rss=int(0.76 * GiB))
with pytest.raises(ValueError) as exc:
worker.determine_available_memory()
msg = str(exc.value)
assert "increase `--gpu-memory-utilization` to more than 0.10" in msg
assert "--kv-cache-memory-bytes" in msg
# Neither the misleading remedy nor a negative KV size.
assert "other processes" not in msg
assert "-0." not in msg
def test_short_node_memory_still_asks_to_free_memory(monkeypatch):
worker = _worker(monkeypatch, reserved=6 * GiB, rss=1 * GiB, available=2 * GiB)
with pytest.raises(ValueError, match="Reduce CPU memory used by other processes"):
worker.determine_available_memory()
def test_budget_that_fits_sizes_the_kv_cache(monkeypatch):
worker = _worker(monkeypatch, reserved=3 * GiB, rss=1 * GiB)
assert worker.determine_available_memory() == 2 * GiB
+217
View File
@@ -0,0 +1,217 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import gc
from types import SimpleNamespace
import pytest
import torch
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization.online.lm_head import (
quantized_lm_head_copy,
refresh_quantized_lm_heads,
)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.reload import (
finalize_layerwise_reload,
initialize_layerwise_reload,
record_metadata_for_reloading,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.platforms import current_platform
from vllm.v1.worker.gpu_worker import Worker
# Not a multiple of the vocab padding, so the head has padded rows.
VOCAB, HIDDEN = 4000, 512
requires_cuda = pytest.mark.skipif(
not torch.cuda.is_available(), reason="Marlin needs CUDA"
)
# NVFP4 needs a CUDA W4A16 kernel; ROCm only runs the fp8 cases.
requires_nvfp4 = pytest.mark.skipif(
not current_platform.is_cuda(), reason="nvfp4 needs a CUDA W4A16 kernel"
)
NVFP4 = pytest.param("nvfp4", marks=requires_nvfp4)
def _target_head(monkeypatch, seed: int = 0) -> ParallelLMHead:
# Online methods read the model dtype from the current config.
model_config = SimpleNamespace(dtype=torch.bfloat16)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.online.fp8.get_current_vllm_config",
lambda: SimpleNamespace(model_config=model_config),
)
head = ParallelLMHead(VOCAB, HIDDEN, disable_tp=True)
gen = torch.Generator().manual_seed(seed)
head.weight.data.copy_(torch.randn(head.weight.shape, generator=gen) * 0.02)
return head.to("cuda", torch.bfloat16)
def _tensors(module: torch.nn.Module) -> dict[str, torch.Tensor]:
return dict(module.named_parameters()) | dict(module.named_buffers())
@requires_cuda
@pytest.mark.parametrize(
("quantization", "max_rel_err", "min_top1"),
[("fp8", 0.05, 0.9), pytest.param("nvfp4", 0.15, 0.75, marks=requires_nvfp4)],
)
def test_tracks_bf16_head_through_logits_processor(
default_vllm_config, dist_init, monkeypatch, quantization, max_rel_err, min_top1
):
head = _target_head(monkeypatch)
target_weight = head.weight.detach().clone()
draft_head = quantized_lm_head_copy(head, quantization)
assert isinstance(draft_head, ParallelLMHead)
# The source head keeps its bf16 weight, and no bf16 copy is kept around.
assert torch.equal(head.weight, target_weight)
assert draft_head.weight.dtype != torch.bfloat16
logits_processor = LogitsProcessor(VOCAB)
hidden = torch.randn(2048, HIDDEN, device="cuda", dtype=torch.bfloat16)
ref = logits_processor(head, hidden).float()
out = logits_processor(draft_head, hidden).float()
assert out.shape == ref.shape == (2048, VOCAB)
rel_err = ((out - ref).norm() / ref.norm()).item()
top1 = (out.argmax(-1) == ref.argmax(-1)).float().mean().item()
assert rel_err < max_rel_err
assert top1 > min_top1
@requires_cuda
@pytest.mark.parametrize("quantization", ["fp8", NVFP4])
def test_refresh_follows_target_update(
default_vllm_config, dist_init, monkeypatch, quantization
):
"""After the target's weights change, the copy is re-derived in place: it
equals a fresh quantization of the new target, keeps its storage, and
frees the temporary copy without waiting for the cycle collector."""
head = _target_head(monkeypatch)
drafter = torch.nn.Module()
drafter.lm_head = quantized_lm_head_copy(head, quantization)
ptrs = {k: t.data_ptr() for k, t in _tensors(drafter.lm_head).items()}
new_weight = _target_head(monkeypatch, seed=1).weight.detach()
head.weight.data.copy_(new_weight)
gc.disable()
try:
allocated = torch.accelerator.memory_allocated()
refresh_quantized_lm_heads(drafter)
assert torch.accelerator.memory_allocated() == allocated
finally:
gc.enable()
expected = _tensors(quantized_lm_head_copy(head, quantization))
for name, tensor in _tensors(drafter.lm_head).items():
assert tensor.data_ptr() == ptrs[name]
if name != "workspace":
assert torch.equal(tensor, expected[name]), name
assert torch.equal(head.weight, new_weight)
def _draft_update_worker(vllm_config, drafter: torch.nn.Module, **engine) -> Worker:
worker = object.__new__(Worker)
worker.vllm_config = vllm_config
worker.weight_transfer_engine = SimpleNamespace(
supports_draft_weight_update=True,
reset_weight_update_target=lambda: None,
**engine,
)
worker.model_runner = SimpleNamespace(get_draft_model=lambda: drafter)
worker._weight_update_active = False
worker._set_draft_weight_update_target = lambda: None
return worker
@requires_cuda
def test_draft_weight_update_refreshes_copy(
default_vllm_config, dist_init, monkeypatch
):
"""A draft weight update can write a target weight the drafter shares, such
as an embedding tied to the lm_head; the copy must follow it."""
head = _target_head(monkeypatch)
drafter = torch.nn.Module()
drafter.lm_head = quantized_lm_head_copy(head, "fp8")
worker = _draft_update_worker(
default_vllm_config,
drafter,
start_weight_update=lambda: None,
finish_weight_update=lambda: None,
)
worker.start_draft_weight_update()
head.weight.data.copy_(_target_head(monkeypatch, seed=1).weight)
worker.finish_weight_update()
expected = _tensors(quantized_lm_head_copy(head, "fp8"))
for name, tensor in _tensors(drafter.lm_head).items():
assert torch.equal(tensor, expected[name]), name
@requires_cuda
@pytest.mark.parametrize("quantization", ["fp8", NVFP4])
def test_draft_weight_update_loads_lm_head(
default_vllm_config, dist_init, monkeypatch, quantization
):
"""A draft update carrying lm_head.weight loads the shared target head, as
without quantization, and the copy is re-derived from it in place."""
head = _target_head(monkeypatch)
record_metadata_for_reloading(head)
drafter = torch.nn.Module()
draft_head = quantized_lm_head_copy(head, quantization)
drafter.lm_head = draft_head
ptrs = {k: t.data_ptr() for k, t in _tensors(draft_head).items()}
new_weight = torch.randn(VOCAB, HIDDEN, device="cuda", dtype=torch.bfloat16)
def load(update_info):
param = drafter.get_parameter("lm_head.weight")
getattr(param, "weight_loader", default_weight_loader)(param, new_weight)
worker = _draft_update_worker(
default_vllm_config,
drafter,
start_weight_update=lambda: initialize_layerwise_reload(drafter),
update_weights=load,
finish_weight_update=lambda: finalize_layerwise_reload(drafter, None),
)
worker.start_draft_weight_update()
worker.update_weights({})
worker.finish_weight_update()
assert drafter.lm_head is draft_head
assert torch.equal(head.weight[:VOCAB], new_weight)
expected = _tensors(quantized_lm_head_copy(head, quantization))
for name, tensor in _tensors(draft_head).items():
assert tensor.data_ptr() == ptrs[name]
if name != "workspace":
assert torch.equal(tensor, expected[name]), name
hidden = torch.randn(64, HIDDEN, device="cuda", dtype=torch.bfloat16)
assert LogitsProcessor(VOCAB)(draft_head, hidden).isfinite().all()
@requires_cuda
@requires_nvfp4
def test_layerwise_reload_of_drafter_keeps_copy(
default_vllm_config, dist_init, monkeypatch
):
"""A drafter weight update reloads the drafter layerwise; the derived copy
must come back unchanged, not be re-quantized."""
head = _target_head(monkeypatch)
drafter = torch.nn.Module()
record_metadata_for_reloading(drafter)
drafter.lm_head = quantized_lm_head_copy(head, "nvfp4")
before = {k: t.clone() for k, t in _tensors(drafter.lm_head).items()}
def fail(*args, **kwargs):
raise AssertionError("the copy was re-quantized")
monkeypatch.setattr(
drafter.lm_head.quant_method, "process_weights_after_loading", fail
)
initialize_layerwise_reload(drafter)
finalize_layerwise_reload(drafter, None)
after = _tensors(drafter.lm_head)
assert after.keys() == before.keys()
for name, tensor in before.items():
assert torch.equal(after[name], tensor), name
+2 -4
View File
@@ -302,10 +302,8 @@ def _sync_dp(
reduced[4] = int(allow_ubatching)
reduced[5] = 8 # num_reqs
with (
patch.object(dp_utils.dist, "all_reduce", lambda t, group: t.copy_(reduced)),
patch.object(dp_utils, "get_dp_group", lambda: SimpleNamespace(cpu_group=None)),
):
dp_group = SimpleNamespace(cpu_all_reduce_=lambda t: t.copy_(reduced))
with patch.object(dp_utils, "get_dp_group", lambda: dp_group):
return dp_utils.sync_cudagraph_and_dp_padding(
cudagraph_manager=cudagraph_manager,
desired_batch_desc=BatchExecutionDescriptor(
@@ -65,6 +65,9 @@ class _RecordingModelRunner:
def reset_lora_state(self) -> None:
self.reset_lora_calls += 1
def get_draft_model(self) -> nn.Module | None:
return None
def _make_worker(engine: _RecordingEngine | None) -> Worker:
worker = object.__new__(Worker)
@@ -62,7 +62,7 @@ def test_mrv2_kv_pool_only_wraps_backing_allocation(monkeypatch, hisparse) -> No
monkeypatch.setattr(
hisparse_binding,
"HiSparseHostPool",
"get_hisparse_host_pool",
lambda vllm_config, kv_cache_config: host_pool,
)
monkeypatch.setattr(hisparse_binding, "allocate_hisparse_kv_caches", allocate)
+5
View File
@@ -1257,6 +1257,8 @@ def test_hisparse_worker_shutdown_releases_pinned_state(monkeypatch):
worker.cache_handles = []
worker.pinned_host_pools = []
worker.shared_host_region = object()
worker.vllm_config = SimpleNamespace(instance_id="shutdown")
pool = hisparse_runtime_module.get_hisparse_host_pool(worker.vllm_config, None)
released = False
def release_pinned_state(runtimes, pinned_host_pools, shared_host_region):
@@ -1273,6 +1275,9 @@ def test_hisparse_worker_shutdown_releases_pinned_state(monkeypatch):
worker.shutdown()
assert released
new_pool = hisparse_runtime_module.get_hisparse_host_pool(worker.vllm_config, None)
assert new_pool is not pool
hisparse_runtime_module.drop_hisparse_host_pool(worker.vllm_config)
class _TestReplaySSMMixer(MambaMixer2):