Merge branch 'main' into patches
This commit is contained in:
@@ -58,7 +58,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
SlidingWindowSpec,
|
||||
SparseCacheRole,
|
||||
UniformTypeKVCacheSpecs,
|
||||
iter_layer_specs,
|
||||
)
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
|
||||
@@ -1945,247 +1944,102 @@ def test_check_enough_kv_cache_memory_reserves_null_block():
|
||||
)
|
||||
|
||||
|
||||
def _spec_decode_grouping_config(method="dspark", model_type=None):
|
||||
"""Grouping config with an EAGLE-family speculative method enabled."""
|
||||
return SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
|
||||
cache_config=SimpleNamespace(
|
||||
get_resolved_kv_cache_layout=lambda: SimpleNamespace(
|
||||
is_block_outermost=True
|
||||
)
|
||||
),
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(model_type=model_type)),
|
||||
speculative_config=SimpleNamespace(
|
||||
method=method,
|
||||
use_eagle=lambda: True,
|
||||
use_eagle_block_drop=lambda: True,
|
||||
),
|
||||
def _spec_decode_kv_cache_groups(worker_specs, use_eagle_block_drop=True):
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=64))
|
||||
config.cache_config.kv_cache_layout = "BLHNC"
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="dflash",
|
||||
use_eagle=lambda: True,
|
||||
use_eagle_block_drop=lambda: use_eagle_block_drop,
|
||||
use_multi_module_mtp=lambda: False,
|
||||
parallel_drafting=False,
|
||||
)
|
||||
configs = get_kv_cache_configs(config, worker_specs, [2**30] * len(worker_specs))
|
||||
return generate_scheduler_kv_cache_config(configs).kv_cache_groups
|
||||
|
||||
|
||||
def _hybrid_specs_with_draft(draft: bool, draft_shares_target_spec: bool = False):
|
||||
"""A K3-shaped hybrid: MLA full attention + Mamba, optionally plus a
|
||||
DSpark-style draft MLA layer marked non_causal_multi_token_decode.
|
||||
def _as_draft(spec):
|
||||
return replace(spec, is_draft=True)
|
||||
|
||||
The target's fp8 KV dtype is one difference from the draft on Kimi-K3
|
||||
(target `--kv-cache-dtype fp8_e4m3`, draft `auto`). Pass
|
||||
draft_shares_target_spec to match that dtype while retaining the draft
|
||||
marker as a group boundary.
|
||||
"""
|
||||
target_dtype = None if draft_shares_target_spec else "fp8_e4m3"
|
||||
specs = {
|
||||
"target.attn.0": new_mla_spec(block_size=64, cache_dtype_str=target_dtype),
|
||||
"target.attn.1": new_mla_spec(block_size=64, cache_dtype_str=target_dtype),
|
||||
"target.mamba.0": new_mamba_spec(block_size=64, mamba_cache_mode="align"),
|
||||
"target.mamba.1": new_mamba_spec(block_size=64, mamba_cache_mode="align"),
|
||||
}
|
||||
|
||||
def _hybrid_specs(draft=True):
|
||||
"""Qwen3.5-shaped [GDN x3, full-attn] x2, plus an MTP full-attn layer."""
|
||||
specs = {}
|
||||
for i in range(8):
|
||||
specs[f"layers.{i}"] = (
|
||||
new_kv_cache_spec()
|
||||
if i % 4 == 3
|
||||
else new_mamba_spec(mamba_cache_mode="align")
|
||||
)
|
||||
if draft:
|
||||
draft_spec = new_mla_spec(block_size=64)
|
||||
specs["draft.attn.0"] = replace(draft_spec, non_causal_multi_token_decode=True)
|
||||
specs["mtp.0"] = _as_draft(new_kv_cache_spec())
|
||||
return specs
|
||||
|
||||
|
||||
def test_draft_group_annotated_on_hybrid_general_path():
|
||||
# A drafter's MLA layer carries non_causal_multi_token_decode, so its group
|
||||
# is identifiable without keying off a model version.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=True)
|
||||
)
|
||||
|
||||
flagged = [g for g in groups if g.is_eagle_group]
|
||||
assert len(flagged) == 1
|
||||
assert "draft.attn.0" in flagged[0].layer_names
|
||||
|
||||
|
||||
def test_mamba_groups_never_flagged_even_when_draft_shares_a_group():
|
||||
# Packed uniform-type groups can contain distinct target and draft layer
|
||||
# specs; the combined group still holds volatile draft KV and must be
|
||||
# flagged. What must never happen is a Mamba group being flagged: that
|
||||
# widens its lookup window to two consecutive chunks, which align-mode
|
||||
# checkpointing never produces, zeroing every lookup.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(),
|
||||
_hybrid_specs_with_draft(draft=True, draft_shares_target_spec=True),
|
||||
)
|
||||
|
||||
for group in groups:
|
||||
if "draft.attn.0" in group.layer_names:
|
||||
assert group.is_eagle_group
|
||||
if isinstance(group.kv_cache_spec, MambaSpec):
|
||||
assert not group.is_eagle_group
|
||||
|
||||
|
||||
def test_draft_group_not_annotated_without_spec_decode():
|
||||
# The marker alone must not flag anything; the eagle semantics only apply
|
||||
# when a speculative method is actually enabled.
|
||||
config = _spec_decode_grouping_config()
|
||||
config.speculative_config = None
|
||||
groups = get_kv_cache_groups(config, _hybrid_specs_with_draft(draft=True))
|
||||
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
|
||||
|
||||
def test_unidentifiable_draft_with_mamba_warns(caplog_vllm):
|
||||
# No group carries the draft marker, so consumers fall back to
|
||||
# conservative behavior that silently breaks reuse for Mamba groups.
|
||||
# That must at least be visible.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=False)
|
||||
)
|
||||
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
assert "no KV cache group could be identified as the draft model's" in (
|
||||
caplog_vllm.text
|
||||
)
|
||||
|
||||
|
||||
def test_unidentifiable_draft_without_mamba_does_not_warn(caplog_vllm):
|
||||
# Pure-attention models degrade gracefully under the consumers'
|
||||
# conservative fallback (a one-block hit drop at most), so the warning
|
||||
# stays silent to avoid noise on every unannotated EAGLE deployment.
|
||||
specs = {
|
||||
"target.attn.0": new_mla_spec(block_size=64),
|
||||
"target.attn.1": new_mla_spec(block_size=64),
|
||||
}
|
||||
groups = get_kv_cache_groups(_spec_decode_grouping_config(), specs)
|
||||
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
assert "could be identified as the draft model's" not in caplog_vllm.text
|
||||
|
||||
|
||||
def test_no_warning_when_draft_group_is_identified(caplog_vllm):
|
||||
get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=True)
|
||||
)
|
||||
|
||||
assert "could be identified as the draft model's" not in caplog_vllm.text
|
||||
|
||||
|
||||
def _deepseek_v4_specs(model_version="deepseek_v4"):
|
||||
"""DeepseekV4-shaped specs: full MLA layers plus sliding-window MLA layers
|
||||
at differing page sizes, with the MTP draft layer registered last."""
|
||||
return {
|
||||
"model.layers.0.self_attn.attn": new_mla_spec(),
|
||||
"model.layers.1.self_attn.attn": new_mla_spec(),
|
||||
"model.layers.2.self_attn.attn": new_swa_mla_spec(
|
||||
head_size=1024, model_version=model_version
|
||||
),
|
||||
# The MTP block registers last, and its sliding-window size differs, so
|
||||
# it lands in a group of its own.
|
||||
"model.layers.3.self_attn.attn": new_swa_mla_spec(
|
||||
head_size=1024, sliding_window=256, model_version=model_version
|
||||
),
|
||||
}
|
||||
_DRAFT_SPEC_SHAPES = {
|
||||
"uniform": lambda: {
|
||||
"layers.0": new_kv_cache_spec(),
|
||||
"draft.0": _as_draft(new_kv_cache_spec()),
|
||||
},
|
||||
"hybrid": _hybrid_specs,
|
||||
"draft_spans_groups": lambda: {
|
||||
"layers.0": new_mla_spec(),
|
||||
"layers.1": new_swa_mla_spec(head_size=1024),
|
||||
"mtp.0": _as_draft(new_mla_spec()),
|
||||
"mtp.1": _as_draft(new_swa_mla_spec(head_size=1024)),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("method", "model_type"),
|
||||
[("mtp", "deepseek_v4"), ("dspark", "deepseek_v4"), ("dspark", "deepseek_v41")],
|
||||
"make_specs", _DRAFT_SPEC_SHAPES.values(), ids=list(_DRAFT_SPEC_SHAPES)
|
||||
)
|
||||
def test_deepseek_v4_draft_group_annotated_on_packed_path(method, model_type):
|
||||
# DeepseekV4's MTP block reuses the target's decoder layer, so its spec
|
||||
# carries no draft marker and only the positional rule can find it. This
|
||||
# pins the pre-existing behaviour that the unified annotator must preserve.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(method=method, model_type=model_type),
|
||||
_deepseek_v4_specs(model_version=None),
|
||||
)
|
||||
|
||||
flagged = [g for g in groups if g.is_eagle_group]
|
||||
assert len(flagged) == 1
|
||||
assert "model.layers.3.self_attn.attn" in flagged[0].layer_names
|
||||
|
||||
|
||||
def test_trailing_layer_fallback_applies_to_any_mtp_model():
|
||||
# The positional rule is sound for every MTP drafter, not just DeepseekV4:
|
||||
# MTP blocks reuse the target's decoder layer (no spec marker) and always
|
||||
# register after every target layer. The model_type must not gate it.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(method="mtp", model_type="other"),
|
||||
_deepseek_v4_specs(),
|
||||
)
|
||||
|
||||
flagged = [g for g in groups if g.is_eagle_group]
|
||||
assert len(flagged) == 1
|
||||
assert "model.layers.3.self_attn.attn" in flagged[0].layer_names
|
||||
|
||||
|
||||
def _qwen3_5_hybrid_specs(with_mtp_layer: bool):
|
||||
"""Qwen3.5-shaped hybrid: repeating [GDN x3, full-attn x1] blocks, with
|
||||
the MTP drafter's full-attn layer (spec-identical to the target's)
|
||||
registered last."""
|
||||
specs = {}
|
||||
idx = 0
|
||||
for _ in range(2):
|
||||
for _ in range(3):
|
||||
specs[f"model.layers.{idx}.linear_attn"] = new_mamba_spec(
|
||||
mamba_cache_mode="align"
|
||||
)
|
||||
idx += 1
|
||||
specs[f"model.layers.{idx}.self_attn.attn"] = new_kv_cache_spec()
|
||||
idx += 1
|
||||
if with_mtp_layer:
|
||||
specs["mtp.layers.0.self_attn.attn"] = new_kv_cache_spec()
|
||||
return specs
|
||||
|
||||
|
||||
def test_qwen3_5_mtp_draft_group_annotated_on_hybrid_path(caplog_vllm):
|
||||
# A hybrid mamba + full-attention model with an MTP drafter that is
|
||||
# spec-indistinguishable from the target reaches the general multi-group
|
||||
# path. The trailing-layer rule must locate the draft group there so the
|
||||
# Mamba groups are not swept up by the flag-all consumer fallback.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(method="mtp", model_type="qwen3_5"),
|
||||
_qwen3_5_hybrid_specs(with_mtp_layer=True),
|
||||
)
|
||||
|
||||
flagged = [g for g in groups if g.is_eagle_group]
|
||||
assert len(flagged) == 1
|
||||
assert "mtp.layers.0.self_attn.attn" in flagged[0].layer_names
|
||||
for group in groups:
|
||||
if any(
|
||||
isinstance(spec, MambaSpec)
|
||||
for spec in iter_layer_specs(group.kv_cache_spec)
|
||||
):
|
||||
assert not group.is_eagle_group
|
||||
assert "could be identified as the draft model's" not in caplog_vllm.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["eagle", "eagle3", "dspark"])
|
||||
def test_non_mtp_eagle_hybrid_still_warns(caplog_vllm, method):
|
||||
# Other drafters are not covered by the trailing-layer rule, so an
|
||||
# unidentifiable hybrid draft must still warn.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(method=method, model_type="qwen3_5"),
|
||||
_qwen3_5_hybrid_specs(with_mtp_layer=True),
|
||||
)
|
||||
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
assert "could be identified as the draft model's" in caplog_vllm.text
|
||||
|
||||
|
||||
def test_trailing_layer_fallback_requires_exact_partition():
|
||||
# If the groups do not partition the layers exactly (e.g. a caller that
|
||||
# dropped or duplicated layers), the positional rule is meaningless and
|
||||
# must not fire.
|
||||
from vllm.v1.core.kv_cache_planning import _annotate_eagle_groups
|
||||
|
||||
specs = _qwen3_5_hybrid_specs(with_mtp_layer=True)
|
||||
config = _spec_decode_grouping_config(method="mtp", model_type="qwen3_5")
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
for g in groups:
|
||||
g.is_eagle_group = False
|
||||
# Remove one layer from its group: no longer an exact partition.
|
||||
trimmed = [
|
||||
KVCacheGroupSpec(
|
||||
[n for n in g.layer_names if n != "model.layers.0.linear_attn"],
|
||||
g.kv_cache_spec,
|
||||
)
|
||||
for g in groups
|
||||
@pytest.mark.parametrize("pp", [1, 2])
|
||||
def test_eagle_groups_are_the_groups_holding_draft_layers(make_specs, pp):
|
||||
specs = make_specs()
|
||||
draft_layers = {name for name, spec in specs.items() if spec.is_draft}
|
||||
untagged = {name: replace(spec, is_draft=False) for name, spec in specs.items()}
|
||||
groups = _spec_decode_kv_cache_groups([specs])
|
||||
assert [(g.layer_names, g.kv_cache_spec) for g in groups] == [
|
||||
(g.layer_names, g.kv_cache_spec)
|
||||
for g in _spec_decode_kv_cache_groups([untagged])
|
||||
]
|
||||
_annotate_eagle_groups(config, specs, trimmed, use_trailing_layer_fallback=True)
|
||||
|
||||
assert not any(g.is_eagle_group for g in trimmed)
|
||||
worker_specs = [
|
||||
{n: s for n, s in specs.items() if n not in draft_layers},
|
||||
{n: s for n, s in specs.items() if n in draft_layers},
|
||||
]
|
||||
scheduler_groups = _spec_decode_kv_cache_groups(
|
||||
[specs] if pp == 1 else worker_specs
|
||||
)
|
||||
assert [g.is_eagle_group for g in scheduler_groups] == [
|
||||
bool(draft_layers & set(g.layer_names)) for g in groups
|
||||
]
|
||||
|
||||
|
||||
def test_no_eagle_groups_without_eagle_block_drop():
|
||||
groups = _spec_decode_kv_cache_groups([_hybrid_specs()], use_eagle_block_drop=False)
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"make_specs,warns",
|
||||
[
|
||||
(lambda: _hybrid_specs(draft=False), True),
|
||||
(
|
||||
lambda: {
|
||||
"layers.0": new_kv_cache_spec(),
|
||||
"draft.0": new_kv_cache_spec(),
|
||||
},
|
||||
False,
|
||||
),
|
||||
],
|
||||
ids=["untagged_hybrid", "untagged_uniform_attention"],
|
||||
)
|
||||
def test_warns_only_when_untagged_draft_meets_mamba(make_specs, warns, caplog_vllm):
|
||||
_spec_decode_kv_cache_groups([make_specs()])
|
||||
warned = "could be identified as the draft model's" in caplog_vllm.text
|
||||
assert warned == warns
|
||||
|
||||
|
||||
_GQA_SPEC = FullAttentionSpec(
|
||||
|
||||
@@ -1303,6 +1303,22 @@ def test_full_attention_merge_preserves_replicated_cache_geometry(use_mla):
|
||||
type(draft).merge([draft, replace(draft, dcp_sharded=True)])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target",
|
||||
[
|
||||
new_kv_cache_spec(),
|
||||
new_mla_spec(),
|
||||
SinkFullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=64, dtype=torch.float32, sink_len=4
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_draft_layer_tag_is_not_part_of_the_cache_format(target):
|
||||
draft = replace(target, is_draft=True)
|
||||
assert draft == target and hash(draft) == hash(target)
|
||||
assert type(target).merge([target, draft]) == target
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"layer_type,dcp_size,expected_width",
|
||||
[
|
||||
|
||||
@@ -929,14 +929,8 @@ class TestPushWriterNotifs:
|
||||
notif_msg = f"{request_id}:1".encode()
|
||||
w._pending_completion_notifs.put(notif_msg)
|
||||
|
||||
# transfer_topo is consulted only for the producer-side path; we
|
||||
# make it a MagicMock because the D-side branch returns early.
|
||||
w.transfer_topo = MagicMock()
|
||||
w._get_new_notifs()
|
||||
|
||||
notified = w._get_new_notifs()
|
||||
|
||||
# Notif consumed; D-side just touches _recving_transfers.
|
||||
assert notified == set()
|
||||
assert request_id in w._recving_transfers
|
||||
|
||||
def test_get_transfer_results_evicts_completed_state(self):
|
||||
@@ -1352,12 +1346,10 @@ class TestPushWriterNegative:
|
||||
logger=("vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker"),
|
||||
)
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.transfer_topo = MagicMock()
|
||||
# Forward a completion notif for an unknown request_id.
|
||||
w._pending_completion_notifs.put(b"never-heard-of-you:1")
|
||||
|
||||
notified = w._get_new_notifs()
|
||||
assert notified == set()
|
||||
w._get_new_notifs()
|
||||
# Did not register anywhere.
|
||||
assert "never-heard-of-you" not in w._recving_transfers
|
||||
assert w.xfer_stats.data["num_notifications_after_expiry"] == [1]
|
||||
@@ -1381,7 +1373,6 @@ class TestPushWriterNegative:
|
||||
leases of tracked P-side requests on the engine main thread, and
|
||||
ignore request IDs that aren't being tracked."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.transfer_topo = MagicMock()
|
||||
# _handle_heartbeat reads ``self._lease_extension`` (set in the
|
||||
# real ``__init__``).
|
||||
w._lease_extension = 10
|
||||
@@ -1395,8 +1386,7 @@ class TestPushWriterNegative:
|
||||
# and another tracked one.
|
||||
w._pending_completion_notifs.put(b"HB:req-a,req-unknown,req-b")
|
||||
|
||||
notified = w._get_new_notifs()
|
||||
assert notified == set()
|
||||
w._get_new_notifs()
|
||||
|
||||
# Tracked leases were renewed strictly forward in time.
|
||||
now = time.perf_counter()
|
||||
@@ -1421,20 +1411,19 @@ class TestPushPipelineParallel:
|
||||
"""Each PP stage WRITEs its own layers and sends one notif; D must
|
||||
collect pp_size notifs before reporting the recv done."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.transfer_topo = MagicMock()
|
||||
request_id = "req-pp-2"
|
||||
w._recving_metadata[request_id] = MagicMock(pp_size=2)
|
||||
notif = f"{request_id}:1".encode()
|
||||
|
||||
# First stage: counted, not yet done.
|
||||
w._pending_completion_notifs.put(notif)
|
||||
assert w._get_new_notifs() == set()
|
||||
w._get_new_notifs()
|
||||
assert request_id not in w._recving_transfers
|
||||
assert w.consumer_notification_counts_by_req[request_id] == 1
|
||||
|
||||
# Second (final) stage: now reported done.
|
||||
w._pending_completion_notifs.put(notif)
|
||||
assert w._get_new_notifs() == set()
|
||||
w._get_new_notifs()
|
||||
assert request_id in w._recving_transfers
|
||||
assert request_id not in w.consumer_notification_counts_by_req
|
||||
|
||||
|
||||
@@ -110,8 +110,8 @@ def test_rocm_aiter_sampler_defers_import_when_generators_force_native(
|
||||
monkeypatch.setattr("builtins.__import__", guard_aiter_sampling_import)
|
||||
|
||||
sampler = topk_topp_sampler.TopKTopPSampler()
|
||||
logits = torch.randn(2, 8)
|
||||
k = torch.full((2,), 2, dtype=torch.int32)
|
||||
logits = torch.randn(2, 8, device=DEVICE_TYPE)
|
||||
k = torch.full((2,), 2, dtype=torch.int32, device=DEVICE_TYPE)
|
||||
generators = {0: torch.Generator(device=logits.device).manual_seed(0)}
|
||||
|
||||
token_ids, logits_to_return = sampler(logits, generators, k, None)
|
||||
|
||||
@@ -103,15 +103,8 @@ def test_mtp_load_model_unified(mock_get_model, mock_get_layers, mock_get_pp_gro
|
||||
|
||||
target_attn_layers = {"target_attn_1": mock.MagicMock()}
|
||||
all_attn_layers = {**target_attn_layers, "draft_attn_1": mock.MagicMock()}
|
||||
target_indexer_layers: dict = {}
|
||||
all_indexer_layers: dict = {}
|
||||
|
||||
mock_get_layers.side_effect = [
|
||||
target_attn_layers,
|
||||
target_indexer_layers,
|
||||
all_attn_layers,
|
||||
all_indexer_layers,
|
||||
]
|
||||
mock_get_layers.side_effect = [target_attn_layers, all_attn_layers]
|
||||
|
||||
mock_pp_group = mock.MagicMock()
|
||||
mock_pp_group.world_size = 1
|
||||
@@ -138,6 +131,8 @@ def test_mtp_load_model_unified(mock_get_model, mock_get_layers, mock_get_pp_gro
|
||||
assert proposer.model.lm_head == target_model.lm_head
|
||||
# MTP shares embed_tokens with target model
|
||||
assert proposer.model.model.embed_tokens == target_model.model.embed_tokens
|
||||
assert proposer._draft_attn_layer_names == {"draft_attn_1"}
|
||||
assert all_attn_layers["draft_attn_1"].is_draft_layer is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_speculative_tokens", [1])
|
||||
|
||||
@@ -9,6 +9,7 @@ from unittest.mock import AsyncMock, MagicMock
|
||||
import pytest
|
||||
|
||||
from vllm.engine.protocol import StreamingInput
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.outputs import STREAM_FINISHED, RequestOutput
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
@@ -107,6 +108,25 @@ def make_output(request_id: str, finished: bool) -> RequestOutput:
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("is_mm_encoder_only", "is_diffusion"),
|
||||
[(True, False), (False, True)],
|
||||
ids=["encoder-only", "diffusion"],
|
||||
)
|
||||
def test_resumable_requests_rejected_for_non_decoder_models(
|
||||
is_mm_encoder_only: bool, is_diffusion: bool
|
||||
):
|
||||
llm = MagicMock()
|
||||
llm.vllm_config.is_mm_encoder_only = is_mm_encoder_only
|
||||
llm.model_config.is_diffusion = is_diffusion
|
||||
|
||||
with pytest.raises(
|
||||
VLLMValidationError,
|
||||
match="Resumable requests are not supported",
|
||||
):
|
||||
AsyncLLM._validate_resumable_request(llm)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_with_async_generator():
|
||||
"""Test generate with an async input generator.
|
||||
@@ -180,6 +200,8 @@ async def test_empty_input_stream_finishes_without_engine_request():
|
||||
llm.get_supported_tasks = AsyncMock(return_value=("generate",))
|
||||
llm._run_output_handler = MagicMock()
|
||||
llm._add_request = AsyncMock()
|
||||
llm.vllm_config = MagicMock(is_mm_encoder_only=False)
|
||||
llm.model_config = MagicMock(is_diffusion=False)
|
||||
|
||||
final_req = MagicMock()
|
||||
final_req.request_id = "request-int"
|
||||
@@ -204,6 +226,7 @@ async def test_empty_input_stream_finishes_without_engine_request():
|
||||
def _make_streaming_llm(num_chunks: int, has_request: bool = True):
|
||||
llm = AsyncLLM.__new__(AsyncLLM)
|
||||
llm._validate_streaming_input_sampling_params = MagicMock()
|
||||
llm._validate_resumable_request = MagicMock()
|
||||
llm.get_supported_tasks = AsyncMock(return_value=("generate",))
|
||||
llm._run_output_handler = MagicMock()
|
||||
llm._add_request = AsyncMock()
|
||||
|
||||
@@ -9,6 +9,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.core.kv_cache_utils as kv_cache_utils
|
||||
from tests.v1.core.utils import create_scheduler as create_async_scheduler
|
||||
from tests.v1.core.utils import create_scheduler as create_spec_scheduler
|
||||
from vllm.config import DeviceConfig, VllmConfig
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -179,6 +180,7 @@ class TestStreamingScheduler(unittest.TestCase):
|
||||
request_id="session",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
)
|
||||
session.append_output_token_ids(10)
|
||||
session.num_computed_tokens = len(session.prompt_token_ids)
|
||||
session.max_tokens = 2
|
||||
|
||||
@@ -195,6 +197,7 @@ class TestStreamingScheduler(unittest.TestCase):
|
||||
assert session.sampling_params.max_tokens == 10
|
||||
assert session.max_tokens == 10
|
||||
|
||||
session.append_output_token_ids(11)
|
||||
session.num_computed_tokens = len(session.prompt_token_ids)
|
||||
|
||||
new_request2 = DummyRequest(
|
||||
@@ -216,6 +219,7 @@ class TestStreamingScheduler(unittest.TestCase):
|
||||
request_id="session",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
)
|
||||
session.append_output_token_ids(10)
|
||||
session.num_computed_tokens = len(session.prompt_token_ids)
|
||||
|
||||
new_request = DummyRequest(
|
||||
@@ -246,6 +250,7 @@ class TestStreamingScheduler(unittest.TestCase):
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
mm_features=[mm_feature],
|
||||
)
|
||||
session.append_output_token_ids(10)
|
||||
session.num_computed_tokens = len(session.prompt_token_ids)
|
||||
|
||||
mm_feature = MultiModalFeatureSpec(
|
||||
@@ -316,6 +321,7 @@ class TestStreamingScheduler(unittest.TestCase):
|
||||
)
|
||||
scheduler.add_request(session)
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
session.append_output_token_ids(10)
|
||||
session.num_computed_tokens = len(session.prompt_token_ids)
|
||||
|
||||
next_request = DummyRequest(
|
||||
@@ -716,3 +722,54 @@ def test_next_chunk_fully_computed_after_stop_on_accepted_draft(async_scheduling
|
||||
|
||||
assert list(session.prompt_token_ids) == [1, 2, 3, 10, 7, 8]
|
||||
assert output.num_scheduled_tokens["session"] == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_queued_before_stop", [False, True])
|
||||
def test_session_resumes_after_stop_while_preempted(chunk_queued_before_stop):
|
||||
"""A session preempted with a step in flight can stop when that step's
|
||||
output is delivered. It must stay schedulable and keep its tokens."""
|
||||
scheduler = create_async_scheduler(async_scheduling=True)
|
||||
session = DummyRequest("session", prompt_token_ids=list(range(10)))
|
||||
scheduler.add_request(session)
|
||||
scheduler.update_from_output(scheduler.schedule(), _model_output("session", [10]))
|
||||
in_flight = scheduler.schedule()
|
||||
next_chunk = DummyRequest("session", prompt_token_ids=[20, 21])
|
||||
if chunk_queued_before_stop:
|
||||
scheduler.add_request(next_chunk)
|
||||
|
||||
scheduler.running.remove(session)
|
||||
scheduler._preempt_request(session, timestamp=0.0)
|
||||
scheduler.update_from_output(in_flight, _model_output("session", [STOP_TOKEN]))
|
||||
# The parked variant holds a streaming-input slot until the next chunk.
|
||||
assert scheduler.num_waiting_for_streaming_input == (
|
||||
0 if chunk_queued_before_stop else 1
|
||||
)
|
||||
if not chunk_queued_before_stop:
|
||||
scheduler.add_request(next_chunk)
|
||||
output = scheduler.schedule()
|
||||
|
||||
assert scheduler.num_waiting_for_streaming_input == 0
|
||||
expected = list(range(10)) + [10, 20, 21]
|
||||
assert list(session.all_token_ids) == expected
|
||||
assert list(session.prompt_token_ids) == expected
|
||||
assert output.num_scheduled_tokens["session"] == len(expected)
|
||||
|
||||
|
||||
def test_non_resumable_request_stays_dequeued_after_stop_while_preempted():
|
||||
"""Only resumable requests continue after a stop; a request that finishes
|
||||
while preempted must not be put back in a queue."""
|
||||
scheduler = create_async_scheduler(async_scheduling=True)
|
||||
request = DummyRequest("req", resumable=False, prompt_token_ids=list(range(10)))
|
||||
scheduler.add_request(request)
|
||||
scheduler.update_from_output(scheduler.schedule(), _model_output("req", [10]))
|
||||
in_flight = scheduler.schedule()
|
||||
|
||||
scheduler.running.remove(request)
|
||||
scheduler._preempt_request(request, timestamp=0.0)
|
||||
scheduler.update_from_output(in_flight, _model_output("req", [STOP_TOKEN]))
|
||||
|
||||
assert request.status == RequestStatus.FINISHED_STOPPED
|
||||
assert request not in scheduler.waiting
|
||||
assert request not in scheduler.kv_holding_waiting
|
||||
assert request not in scheduler.deferred_waiting
|
||||
assert "req" not in scheduler.schedule().num_scheduled_tokens
|
||||
|
||||
@@ -92,6 +92,7 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
|
||||
layers[name] = SimpleNamespace(
|
||||
get_kv_cache_spec=lambda _, spec=specs[name]: spec,
|
||||
get_attn_backend=lambda backend=backend: backend,
|
||||
is_draft_layer=False,
|
||||
)
|
||||
monkeypatch.setattr(attn_utils, "get_layers_from_vllm_config", lambda *_: layers)
|
||||
monkeypatch.setattr(
|
||||
@@ -112,6 +113,29 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
|
||||
assert all(spec.block_size == block_size for spec in specs.values())
|
||||
|
||||
|
||||
def test_get_kv_cache_spec_tags_draft_layers(monkeypatch):
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=128, dtype=torch.bfloat16
|
||||
)
|
||||
backend = SimpleNamespace(
|
||||
customize_spec=AttentionBackend.customize_spec,
|
||||
get_supported_kernel_block_sizes=lambda _=None: [16],
|
||||
)
|
||||
layers = {
|
||||
name: SimpleNamespace(
|
||||
get_kv_cache_spec=lambda _: spec,
|
||||
get_attn_backend=lambda: backend,
|
||||
is_draft_layer=name == "draft",
|
||||
)
|
||||
for name in ("target", "draft")
|
||||
}
|
||||
monkeypatch.setattr(attn_utils, "get_layers_from_vllm_config", lambda *_: layers)
|
||||
config = SimpleNamespace(attention_config=SimpleNamespace(hisparse_config=None))
|
||||
|
||||
specs = attn_utils.get_kv_cache_spec(config)
|
||||
assert specs["draft"].is_draft and not specs["target"].is_draft
|
||||
|
||||
|
||||
class _FakeMetadataBuilder:
|
||||
def __init__(self, support: AttentionCGSupport, varlen_bound: int | None = None):
|
||||
self.support = support
|
||||
|
||||
@@ -305,3 +305,104 @@ def test_dummy_request_slot_mapping_is_pad():
|
||||
num_tokens_padded=3,
|
||||
)
|
||||
assert dummy[0].tolist() == [PAD_SLOT_ID] * 3
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def populated_block_tables():
|
||||
tables = BlockTables([16, 32], 8, 64, [3, 5], torch.device("cuda"))
|
||||
for req_index in range(8):
|
||||
tables.append_block_ids(
|
||||
req_index, ([req_index + 1] * 3, [req_index + 1] * 5), overwrite=True
|
||||
)
|
||||
tables.apply_staged_writes()
|
||||
tables.gather_block_tables(torch.arange(8, device="cuda"), num_reqs_padded=8)
|
||||
return tables
|
||||
|
||||
|
||||
@pytest.mark.parametrize("previous_dummy", [False, True])
|
||||
@pytest.mark.parametrize("next_dummy", [False, True])
|
||||
def test_shared_block_tables_clear_rows_from_larger_batch(
|
||||
populated_block_tables, previous_dummy, next_dummy
|
||||
):
|
||||
"""A smaller target batch must leave no stale rows for larger draft graphs."""
|
||||
tables = populated_block_tables
|
||||
if previous_dummy:
|
||||
for table in tables.get_dummy_block_tables(8):
|
||||
table.fill_(99) # Profiling supplies valid context/state slots.
|
||||
ptrs = [table.data_ptr() for table in tables.input_block_tables]
|
||||
idx = torch.tensor([7, 2, 0, 5, 1], device="cuda")
|
||||
for num_reqs in (3, 5, 0):
|
||||
if next_dummy:
|
||||
out = tables.get_dummy_block_tables(num_reqs)
|
||||
num_rows = num_reqs
|
||||
else:
|
||||
num_rows = num_reqs + 1
|
||||
out = tables.gather_block_tables(idx[:num_reqs], num_rows)
|
||||
for table, view, ptr in zip(tables.input_block_tables, out, ptrs):
|
||||
assert view.shape == (num_rows, table.shape[1])
|
||||
assert table.data_ptr() == ptr
|
||||
if num_rows:
|
||||
assert view.data_ptr() == ptr
|
||||
assert torch.count_nonzero(table[num_reqs:]).item() == 0
|
||||
if next_dummy:
|
||||
assert torch.count_nonzero(view).item() == 0
|
||||
view.fill_(99)
|
||||
else:
|
||||
expected = (idx[:num_reqs, None] + 1).expand(-1, table.shape[1])
|
||||
torch.testing.assert_close(view[:num_reqs], expected.to(torch.int32))
|
||||
|
||||
|
||||
def test_external_block_table_output_preserves_shared_rows(populated_block_tables):
|
||||
"""PCP-local gathers must neither clear shared rows nor reset their tracking."""
|
||||
tables = populated_block_tables
|
||||
previous = [table.clone() for table in tables.input_block_tables]
|
||||
out = tuple(torch.full_like(table[:3], -1) for table in previous)
|
||||
out_ptrs = torch.tensor(
|
||||
[table.data_ptr() for table in out], dtype=torch.uint64, device="cuda"
|
||||
)
|
||||
idx = torch.tensor([7, 2], device="cuda")
|
||||
result = tables.gather_block_tables(idx, 3, out=out, out_ptrs=out_ptrs)
|
||||
for table, before, view in zip(tables.input_block_tables, previous, result):
|
||||
torch.testing.assert_close(table, before)
|
||||
assert view[:, 0].tolist() == [8, 3, 0]
|
||||
tables.gather_block_tables(idx[:1], num_reqs_padded=1)
|
||||
for table in tables.input_block_tables:
|
||||
assert torch.count_nonzero(table[1:]).item() == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_cuda_graph", [False, True])
|
||||
def test_larger_draft_batch_uses_null_kpool_slots(
|
||||
populated_block_tables, use_cuda_graph
|
||||
):
|
||||
"""Draft padding must not resolve to another request's physical KPool slots."""
|
||||
from vllm.v1.attention.backends.mla.indexer import compute_kpool_tail_slot_mapping
|
||||
|
||||
tables = populated_block_tables
|
||||
idx = torch.tensor([2, 0, 4, 1, 3], device="cuda")
|
||||
query_start_loc = torch.arange(9, dtype=torch.int32, device="cuda").clamp_max_(5)
|
||||
slot_mapping = torch.full((8,), -1, dtype=torch.int64, device="cuda")
|
||||
positions = torch.arange(8, dtype=torch.int64, device="cuda")
|
||||
|
||||
def compute_slots():
|
||||
return compute_kpool_tail_slot_mapping(
|
||||
slot_mapping,
|
||||
tables.input_block_tables[0],
|
||||
query_start_loc,
|
||||
positions,
|
||||
num_actual_tokens=8,
|
||||
num_reqs=8,
|
||||
kpool=8,
|
||||
)
|
||||
|
||||
if use_cuda_graph:
|
||||
compute_slots()
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
slots = compute_slots()
|
||||
tables.gather_block_tables(idx, num_reqs_padded=5)
|
||||
if use_cuda_graph:
|
||||
graph.replay()
|
||||
else:
|
||||
slots = compute_slots()
|
||||
assert slots.tolist() == [24, 9, 42, 19, 36, -1, -1, -1]
|
||||
|
||||
@@ -22,7 +22,12 @@ def _make_connector(
|
||||
backend.bind_connector_metadata.side_effect = lambda _: events.append("bind")
|
||||
backend.start_load_kv.side_effect = lambda *_args, **_kwargs: events.append("start")
|
||||
backend.wait_for_save.side_effect = lambda: events.append("wait")
|
||||
backend.get_transfer_results.return_value = KVConnectorTransferResults()
|
||||
|
||||
def get_transfer_results(_finished_req_ids):
|
||||
events.append("results")
|
||||
return KVConnectorTransferResults()
|
||||
|
||||
backend.get_transfer_results.side_effect = get_transfer_results
|
||||
backend.get_block_ids_with_load_errors.return_value = set()
|
||||
backend.get_kv_connector_stats.return_value = None
|
||||
backend.get_kv_connector_kv_cache_events.return_value = None
|
||||
@@ -73,7 +78,7 @@ def test_load_start_phase(
|
||||
)
|
||||
|
||||
connector.post_forward(set())
|
||||
assert events == ["handle", "bind", "start", "wait", "clear"]
|
||||
assert events == ["handle", "bind", "start", "wait", "results", "clear"]
|
||||
|
||||
kwargs = connector.kv_connector.start_load_kv.call_args.kwargs
|
||||
assert kwargs["request_state_indices"] is request_indices
|
||||
@@ -94,4 +99,21 @@ def test_no_forward_starts_deferred_load_once(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
connector.no_forward(_scheduler_output(False))
|
||||
|
||||
assert events == ["handle", "bind", "start", "wait", "clear"]
|
||||
assert events == ["handle", "bind", "start", "wait", "results", "clear"]
|
||||
|
||||
|
||||
def test_deferred_finalize_collects_results_before_saving(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""With a drafter, the transfer results are collected right after the
|
||||
target forward, and the saves and metadata are finalized after the
|
||||
drafter writes its KV."""
|
||||
events: list[str] = []
|
||||
connector = _make_connector(monkeypatch, events)
|
||||
connector.pre_forward(_scheduler_output(False))
|
||||
|
||||
connector.post_forward(set(), defer_finalize=True)
|
||||
assert events == ["handle", "bind", "start", "results"]
|
||||
|
||||
connector.finalize_step()
|
||||
assert events == ["handle", "bind", "start", "results", "wait", "clear"]
|
||||
|
||||
@@ -88,6 +88,7 @@ def _make_runner(
|
||||
"""Stub model runner exposing only what the warmup entry points read."""
|
||||
return SimpleNamespace(
|
||||
num_speculative_steps=num_spec_steps,
|
||||
speculative_config=SimpleNamespace() if num_spec_steps else None,
|
||||
adaptive_verification=None,
|
||||
rejection_sampler=None,
|
||||
decode_query_len=num_spec_steps + 1,
|
||||
|
||||
@@ -74,6 +74,7 @@ def test_kernel_warmup_restores_uncalibrated_adaptive_manager(monkeypatch, fail_
|
||||
runner = SimpleNamespace(
|
||||
adaptive_verification=manager,
|
||||
rejection_sampler=rejection_sampler,
|
||||
speculative_config=SimpleNamespace(),
|
||||
)
|
||||
|
||||
def run_steps(model_runner, execute, sample, greedy=False):
|
||||
@@ -91,3 +92,27 @@ def test_kernel_warmup_restores_uncalibrated_adaptive_manager(monkeypatch, fail_
|
||||
assert runner.adaptive_verification is manager
|
||||
assert manager.cost_tables is None
|
||||
assert rejection_sampler.enable_adaptive_verification
|
||||
|
||||
|
||||
@pytest.mark.parametrize("speculative", [False, True])
|
||||
def test_kernel_warmup_runs_same_rounds_on_every_pp_stage(monkeypatch, speculative):
|
||||
"""PP receivers need a matching sender for every speculative warmup round."""
|
||||
rounds = []
|
||||
monkeypatch.setattr(
|
||||
warmup,
|
||||
"_warmup_kernels",
|
||||
lambda runner, execute, sample, greedy=False: rounds.append(greedy),
|
||||
)
|
||||
for is_last_pp_rank in (False, True):
|
||||
rounds.clear()
|
||||
runner = SimpleNamespace(
|
||||
adaptive_verification=None,
|
||||
speculative_config=SimpleNamespace() if speculative else None,
|
||||
rejection_sampler=(
|
||||
SimpleNamespace(enable_adaptive_verification=False)
|
||||
if is_last_pp_rank and speculative
|
||||
else None
|
||||
),
|
||||
)
|
||||
warmup.warmup_kernels(runner, _fail, _fail)
|
||||
assert rounds == ([False, True] if speculative else [False])
|
||||
|
||||
Reference in New Issue
Block a user