Merge branch 'main' into patches

This commit is contained in:
2026-10-18 16:04:36 +07:00
299 changed files with 51483 additions and 2195 deletions
+82 -228
View File
@@ -58,7 +58,6 @@ from vllm.v1.kv_cache_interface import (
SlidingWindowSpec,
SparseCacheRole,
UniformTypeKVCacheSpecs,
iter_layer_specs,
)
from vllm.v1.kv_cache_layout import KVCacheLayout
@@ -1945,247 +1944,102 @@ def test_check_enough_kv_cache_memory_reserves_null_block():
)
def _spec_decode_grouping_config(method="dspark", model_type=None):
"""Grouping config with an EAGLE-family speculative method enabled."""
return SimpleNamespace(
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
cache_config=SimpleNamespace(
get_resolved_kv_cache_layout=lambda: SimpleNamespace(
is_block_outermost=True
)
),
model_config=SimpleNamespace(hf_config=SimpleNamespace(model_type=model_type)),
speculative_config=SimpleNamespace(
method=method,
use_eagle=lambda: True,
use_eagle_block_drop=lambda: True,
),
def _spec_decode_kv_cache_groups(worker_specs, use_eagle_block_drop=True):
config = VllmConfig(model_config=ModelConfig(max_model_len=64))
config.cache_config.kv_cache_layout = "BLHNC"
config.speculative_config = SimpleNamespace(
method="dflash",
use_eagle=lambda: True,
use_eagle_block_drop=lambda: use_eagle_block_drop,
use_multi_module_mtp=lambda: False,
parallel_drafting=False,
)
configs = get_kv_cache_configs(config, worker_specs, [2**30] * len(worker_specs))
return generate_scheduler_kv_cache_config(configs).kv_cache_groups
def _hybrid_specs_with_draft(draft: bool, draft_shares_target_spec: bool = False):
"""A K3-shaped hybrid: MLA full attention + Mamba, optionally plus a
DSpark-style draft MLA layer marked non_causal_multi_token_decode.
def _as_draft(spec):
return replace(spec, is_draft=True)
The target's fp8 KV dtype is one difference from the draft on Kimi-K3
(target `--kv-cache-dtype fp8_e4m3`, draft `auto`). Pass
draft_shares_target_spec to match that dtype while retaining the draft
marker as a group boundary.
"""
target_dtype = None if draft_shares_target_spec else "fp8_e4m3"
specs = {
"target.attn.0": new_mla_spec(block_size=64, cache_dtype_str=target_dtype),
"target.attn.1": new_mla_spec(block_size=64, cache_dtype_str=target_dtype),
"target.mamba.0": new_mamba_spec(block_size=64, mamba_cache_mode="align"),
"target.mamba.1": new_mamba_spec(block_size=64, mamba_cache_mode="align"),
}
def _hybrid_specs(draft=True):
"""Qwen3.5-shaped [GDN x3, full-attn] x2, plus an MTP full-attn layer."""
specs = {}
for i in range(8):
specs[f"layers.{i}"] = (
new_kv_cache_spec()
if i % 4 == 3
else new_mamba_spec(mamba_cache_mode="align")
)
if draft:
draft_spec = new_mla_spec(block_size=64)
specs["draft.attn.0"] = replace(draft_spec, non_causal_multi_token_decode=True)
specs["mtp.0"] = _as_draft(new_kv_cache_spec())
return specs
def test_draft_group_annotated_on_hybrid_general_path():
# A drafter's MLA layer carries non_causal_multi_token_decode, so its group
# is identifiable without keying off a model version.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=True)
)
flagged = [g for g in groups if g.is_eagle_group]
assert len(flagged) == 1
assert "draft.attn.0" in flagged[0].layer_names
def test_mamba_groups_never_flagged_even_when_draft_shares_a_group():
# Packed uniform-type groups can contain distinct target and draft layer
# specs; the combined group still holds volatile draft KV and must be
# flagged. What must never happen is a Mamba group being flagged: that
# widens its lookup window to two consecutive chunks, which align-mode
# checkpointing never produces, zeroing every lookup.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(),
_hybrid_specs_with_draft(draft=True, draft_shares_target_spec=True),
)
for group in groups:
if "draft.attn.0" in group.layer_names:
assert group.is_eagle_group
if isinstance(group.kv_cache_spec, MambaSpec):
assert not group.is_eagle_group
def test_draft_group_not_annotated_without_spec_decode():
# The marker alone must not flag anything; the eagle semantics only apply
# when a speculative method is actually enabled.
config = _spec_decode_grouping_config()
config.speculative_config = None
groups = get_kv_cache_groups(config, _hybrid_specs_with_draft(draft=True))
assert not any(g.is_eagle_group for g in groups)
def test_unidentifiable_draft_with_mamba_warns(caplog_vllm):
# No group carries the draft marker, so consumers fall back to
# conservative behavior that silently breaks reuse for Mamba groups.
# That must at least be visible.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=False)
)
assert not any(g.is_eagle_group for g in groups)
assert "no KV cache group could be identified as the draft model's" in (
caplog_vllm.text
)
def test_unidentifiable_draft_without_mamba_does_not_warn(caplog_vllm):
# Pure-attention models degrade gracefully under the consumers'
# conservative fallback (a one-block hit drop at most), so the warning
# stays silent to avoid noise on every unannotated EAGLE deployment.
specs = {
"target.attn.0": new_mla_spec(block_size=64),
"target.attn.1": new_mla_spec(block_size=64),
}
groups = get_kv_cache_groups(_spec_decode_grouping_config(), specs)
assert not any(g.is_eagle_group for g in groups)
assert "could be identified as the draft model's" not in caplog_vllm.text
def test_no_warning_when_draft_group_is_identified(caplog_vllm):
get_kv_cache_groups(
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=True)
)
assert "could be identified as the draft model's" not in caplog_vllm.text
def _deepseek_v4_specs(model_version="deepseek_v4"):
"""DeepseekV4-shaped specs: full MLA layers plus sliding-window MLA layers
at differing page sizes, with the MTP draft layer registered last."""
return {
"model.layers.0.self_attn.attn": new_mla_spec(),
"model.layers.1.self_attn.attn": new_mla_spec(),
"model.layers.2.self_attn.attn": new_swa_mla_spec(
head_size=1024, model_version=model_version
),
# The MTP block registers last, and its sliding-window size differs, so
# it lands in a group of its own.
"model.layers.3.self_attn.attn": new_swa_mla_spec(
head_size=1024, sliding_window=256, model_version=model_version
),
}
_DRAFT_SPEC_SHAPES = {
"uniform": lambda: {
"layers.0": new_kv_cache_spec(),
"draft.0": _as_draft(new_kv_cache_spec()),
},
"hybrid": _hybrid_specs,
"draft_spans_groups": lambda: {
"layers.0": new_mla_spec(),
"layers.1": new_swa_mla_spec(head_size=1024),
"mtp.0": _as_draft(new_mla_spec()),
"mtp.1": _as_draft(new_swa_mla_spec(head_size=1024)),
},
}
@pytest.mark.parametrize(
("method", "model_type"),
[("mtp", "deepseek_v4"), ("dspark", "deepseek_v4"), ("dspark", "deepseek_v41")],
"make_specs", _DRAFT_SPEC_SHAPES.values(), ids=list(_DRAFT_SPEC_SHAPES)
)
def test_deepseek_v4_draft_group_annotated_on_packed_path(method, model_type):
# DeepseekV4's MTP block reuses the target's decoder layer, so its spec
# carries no draft marker and only the positional rule can find it. This
# pins the pre-existing behaviour that the unified annotator must preserve.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(method=method, model_type=model_type),
_deepseek_v4_specs(model_version=None),
)
flagged = [g for g in groups if g.is_eagle_group]
assert len(flagged) == 1
assert "model.layers.3.self_attn.attn" in flagged[0].layer_names
def test_trailing_layer_fallback_applies_to_any_mtp_model():
# The positional rule is sound for every MTP drafter, not just DeepseekV4:
# MTP blocks reuse the target's decoder layer (no spec marker) and always
# register after every target layer. The model_type must not gate it.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(method="mtp", model_type="other"),
_deepseek_v4_specs(),
)
flagged = [g for g in groups if g.is_eagle_group]
assert len(flagged) == 1
assert "model.layers.3.self_attn.attn" in flagged[0].layer_names
def _qwen3_5_hybrid_specs(with_mtp_layer: bool):
"""Qwen3.5-shaped hybrid: repeating [GDN x3, full-attn x1] blocks, with
the MTP drafter's full-attn layer (spec-identical to the target's)
registered last."""
specs = {}
idx = 0
for _ in range(2):
for _ in range(3):
specs[f"model.layers.{idx}.linear_attn"] = new_mamba_spec(
mamba_cache_mode="align"
)
idx += 1
specs[f"model.layers.{idx}.self_attn.attn"] = new_kv_cache_spec()
idx += 1
if with_mtp_layer:
specs["mtp.layers.0.self_attn.attn"] = new_kv_cache_spec()
return specs
def test_qwen3_5_mtp_draft_group_annotated_on_hybrid_path(caplog_vllm):
# A hybrid mamba + full-attention model with an MTP drafter that is
# spec-indistinguishable from the target reaches the general multi-group
# path. The trailing-layer rule must locate the draft group there so the
# Mamba groups are not swept up by the flag-all consumer fallback.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(method="mtp", model_type="qwen3_5"),
_qwen3_5_hybrid_specs(with_mtp_layer=True),
)
flagged = [g for g in groups if g.is_eagle_group]
assert len(flagged) == 1
assert "mtp.layers.0.self_attn.attn" in flagged[0].layer_names
for group in groups:
if any(
isinstance(spec, MambaSpec)
for spec in iter_layer_specs(group.kv_cache_spec)
):
assert not group.is_eagle_group
assert "could be identified as the draft model's" not in caplog_vllm.text
@pytest.mark.parametrize("method", ["eagle", "eagle3", "dspark"])
def test_non_mtp_eagle_hybrid_still_warns(caplog_vllm, method):
# Other drafters are not covered by the trailing-layer rule, so an
# unidentifiable hybrid draft must still warn.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(method=method, model_type="qwen3_5"),
_qwen3_5_hybrid_specs(with_mtp_layer=True),
)
assert not any(g.is_eagle_group for g in groups)
assert "could be identified as the draft model's" in caplog_vllm.text
def test_trailing_layer_fallback_requires_exact_partition():
# If the groups do not partition the layers exactly (e.g. a caller that
# dropped or duplicated layers), the positional rule is meaningless and
# must not fire.
from vllm.v1.core.kv_cache_planning import _annotate_eagle_groups
specs = _qwen3_5_hybrid_specs(with_mtp_layer=True)
config = _spec_decode_grouping_config(method="mtp", model_type="qwen3_5")
groups = get_kv_cache_groups(config, specs)
for g in groups:
g.is_eagle_group = False
# Remove one layer from its group: no longer an exact partition.
trimmed = [
KVCacheGroupSpec(
[n for n in g.layer_names if n != "model.layers.0.linear_attn"],
g.kv_cache_spec,
)
for g in groups
@pytest.mark.parametrize("pp", [1, 2])
def test_eagle_groups_are_the_groups_holding_draft_layers(make_specs, pp):
specs = make_specs()
draft_layers = {name for name, spec in specs.items() if spec.is_draft}
untagged = {name: replace(spec, is_draft=False) for name, spec in specs.items()}
groups = _spec_decode_kv_cache_groups([specs])
assert [(g.layer_names, g.kv_cache_spec) for g in groups] == [
(g.layer_names, g.kv_cache_spec)
for g in _spec_decode_kv_cache_groups([untagged])
]
_annotate_eagle_groups(config, specs, trimmed, use_trailing_layer_fallback=True)
assert not any(g.is_eagle_group for g in trimmed)
worker_specs = [
{n: s for n, s in specs.items() if n not in draft_layers},
{n: s for n, s in specs.items() if n in draft_layers},
]
scheduler_groups = _spec_decode_kv_cache_groups(
[specs] if pp == 1 else worker_specs
)
assert [g.is_eagle_group for g in scheduler_groups] == [
bool(draft_layers & set(g.layer_names)) for g in groups
]
def test_no_eagle_groups_without_eagle_block_drop():
groups = _spec_decode_kv_cache_groups([_hybrid_specs()], use_eagle_block_drop=False)
assert not any(g.is_eagle_group for g in groups)
@pytest.mark.parametrize(
"make_specs,warns",
[
(lambda: _hybrid_specs(draft=False), True),
(
lambda: {
"layers.0": new_kv_cache_spec(),
"draft.0": new_kv_cache_spec(),
},
False,
),
],
ids=["untagged_hybrid", "untagged_uniform_attention"],
)
def test_warns_only_when_untagged_draft_meets_mamba(make_specs, warns, caplog_vllm):
_spec_decode_kv_cache_groups([make_specs()])
warned = "could be identified as the draft model's" in caplog_vllm.text
assert warned == warns
_GQA_SPEC = FullAttentionSpec(
+16
View File
@@ -1303,6 +1303,22 @@ def test_full_attention_merge_preserves_replicated_cache_geometry(use_mla):
type(draft).merge([draft, replace(draft, dcp_sharded=True)])
@pytest.mark.parametrize(
"target",
[
new_kv_cache_spec(),
new_mla_spec(),
SinkFullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=64, dtype=torch.float32, sink_len=4
),
],
)
def test_draft_layer_tag_is_not_part_of_the_cache_format(target):
draft = replace(target, is_draft=True)
assert draft == target and hash(draft) == hash(target)
assert type(target).merge([target, draft]) == target
@pytest.mark.parametrize(
"layer_type,dcp_size,expected_width",
[
@@ -929,14 +929,8 @@ class TestPushWriterNotifs:
notif_msg = f"{request_id}:1".encode()
w._pending_completion_notifs.put(notif_msg)
# transfer_topo is consulted only for the producer-side path; we
# make it a MagicMock because the D-side branch returns early.
w.transfer_topo = MagicMock()
w._get_new_notifs()
notified = w._get_new_notifs()
# Notif consumed; D-side just touches _recving_transfers.
assert notified == set()
assert request_id in w._recving_transfers
def test_get_transfer_results_evicts_completed_state(self):
@@ -1352,12 +1346,10 @@ class TestPushWriterNegative:
logger=("vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker"),
)
w = _StubWriterWorker.fresh()
w.transfer_topo = MagicMock()
# Forward a completion notif for an unknown request_id.
w._pending_completion_notifs.put(b"never-heard-of-you:1")
notified = w._get_new_notifs()
assert notified == set()
w._get_new_notifs()
# Did not register anywhere.
assert "never-heard-of-you" not in w._recving_transfers
assert w.xfer_stats.data["num_notifications_after_expiry"] == [1]
@@ -1381,7 +1373,6 @@ class TestPushWriterNegative:
leases of tracked P-side requests on the engine main thread, and
ignore request IDs that aren't being tracked."""
w = _StubWriterWorker.fresh()
w.transfer_topo = MagicMock()
# _handle_heartbeat reads ``self._lease_extension`` (set in the
# real ``__init__``).
w._lease_extension = 10
@@ -1395,8 +1386,7 @@ class TestPushWriterNegative:
# and another tracked one.
w._pending_completion_notifs.put(b"HB:req-a,req-unknown,req-b")
notified = w._get_new_notifs()
assert notified == set()
w._get_new_notifs()
# Tracked leases were renewed strictly forward in time.
now = time.perf_counter()
@@ -1421,20 +1411,19 @@ class TestPushPipelineParallel:
"""Each PP stage WRITEs its own layers and sends one notif; D must
collect pp_size notifs before reporting the recv done."""
w = _StubWriterWorker.fresh()
w.transfer_topo = MagicMock()
request_id = "req-pp-2"
w._recving_metadata[request_id] = MagicMock(pp_size=2)
notif = f"{request_id}:1".encode()
# First stage: counted, not yet done.
w._pending_completion_notifs.put(notif)
assert w._get_new_notifs() == set()
w._get_new_notifs()
assert request_id not in w._recving_transfers
assert w.consumer_notification_counts_by_req[request_id] == 1
# Second (final) stage: now reported done.
w._pending_completion_notifs.put(notif)
assert w._get_new_notifs() == set()
w._get_new_notifs()
assert request_id in w._recving_transfers
assert request_id not in w.consumer_notification_counts_by_req
+2 -2
View File
@@ -110,8 +110,8 @@ def test_rocm_aiter_sampler_defers_import_when_generators_force_native(
monkeypatch.setattr("builtins.__import__", guard_aiter_sampling_import)
sampler = topk_topp_sampler.TopKTopPSampler()
logits = torch.randn(2, 8)
k = torch.full((2,), 2, dtype=torch.int32)
logits = torch.randn(2, 8, device=DEVICE_TYPE)
k = torch.full((2,), 2, dtype=torch.int32, device=DEVICE_TYPE)
generators = {0: torch.Generator(device=logits.device).manual_seed(0)}
token_ids, logits_to_return = sampler(logits, generators, k, None)
+3 -8
View File
@@ -103,15 +103,8 @@ def test_mtp_load_model_unified(mock_get_model, mock_get_layers, mock_get_pp_gro
target_attn_layers = {"target_attn_1": mock.MagicMock()}
all_attn_layers = {**target_attn_layers, "draft_attn_1": mock.MagicMock()}
target_indexer_layers: dict = {}
all_indexer_layers: dict = {}
mock_get_layers.side_effect = [
target_attn_layers,
target_indexer_layers,
all_attn_layers,
all_indexer_layers,
]
mock_get_layers.side_effect = [target_attn_layers, all_attn_layers]
mock_pp_group = mock.MagicMock()
mock_pp_group.world_size = 1
@@ -138,6 +131,8 @@ def test_mtp_load_model_unified(mock_get_model, mock_get_layers, mock_get_pp_gro
assert proposer.model.lm_head == target_model.lm_head
# MTP shares embed_tokens with target model
assert proposer.model.model.embed_tokens == target_model.model.embed_tokens
assert proposer._draft_attn_layer_names == {"draft_attn_1"}
assert all_attn_layers["draft_attn_1"].is_draft_layer is True
@pytest.mark.parametrize("num_speculative_tokens", [1])
@@ -9,6 +9,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from vllm.engine.protocol import StreamingInput
from vllm.exceptions import VLLMValidationError
from vllm.outputs import STREAM_FINISHED, RequestOutput
from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.v1.engine.async_llm import AsyncLLM
@@ -107,6 +108,25 @@ def make_output(request_id: str, finished: bool) -> RequestOutput:
)
@pytest.mark.parametrize(
("is_mm_encoder_only", "is_diffusion"),
[(True, False), (False, True)],
ids=["encoder-only", "diffusion"],
)
def test_resumable_requests_rejected_for_non_decoder_models(
is_mm_encoder_only: bool, is_diffusion: bool
):
llm = MagicMock()
llm.vllm_config.is_mm_encoder_only = is_mm_encoder_only
llm.model_config.is_diffusion = is_diffusion
with pytest.raises(
VLLMValidationError,
match="Resumable requests are not supported",
):
AsyncLLM._validate_resumable_request(llm)
@pytest.mark.asyncio
async def test_generate_with_async_generator():
"""Test generate with an async input generator.
@@ -180,6 +200,8 @@ async def test_empty_input_stream_finishes_without_engine_request():
llm.get_supported_tasks = AsyncMock(return_value=("generate",))
llm._run_output_handler = MagicMock()
llm._add_request = AsyncMock()
llm.vllm_config = MagicMock(is_mm_encoder_only=False)
llm.model_config = MagicMock(is_diffusion=False)
final_req = MagicMock()
final_req.request_id = "request-int"
@@ -204,6 +226,7 @@ async def test_empty_input_stream_finishes_without_engine_request():
def _make_streaming_llm(num_chunks: int, has_request: bool = True):
llm = AsyncLLM.__new__(AsyncLLM)
llm._validate_streaming_input_sampling_params = MagicMock()
llm._validate_resumable_request = MagicMock()
llm.get_supported_tasks = AsyncMock(return_value=("generate",))
llm._run_output_handler = MagicMock()
llm._add_request = AsyncMock()
@@ -9,6 +9,7 @@ import pytest
import torch
import vllm.v1.core.kv_cache_utils as kv_cache_utils
from tests.v1.core.utils import create_scheduler as create_async_scheduler
from tests.v1.core.utils import create_scheduler as create_spec_scheduler
from vllm.config import DeviceConfig, VllmConfig
from vllm.multimodal.inputs import (
@@ -179,6 +180,7 @@ class TestStreamingScheduler(unittest.TestCase):
request_id="session",
prompt_token_ids=[1, 2, 3],
)
session.append_output_token_ids(10)
session.num_computed_tokens = len(session.prompt_token_ids)
session.max_tokens = 2
@@ -195,6 +197,7 @@ class TestStreamingScheduler(unittest.TestCase):
assert session.sampling_params.max_tokens == 10
assert session.max_tokens == 10
session.append_output_token_ids(11)
session.num_computed_tokens = len(session.prompt_token_ids)
new_request2 = DummyRequest(
@@ -216,6 +219,7 @@ class TestStreamingScheduler(unittest.TestCase):
request_id="session",
prompt_token_ids=[1, 2, 3],
)
session.append_output_token_ids(10)
session.num_computed_tokens = len(session.prompt_token_ids)
new_request = DummyRequest(
@@ -246,6 +250,7 @@ class TestStreamingScheduler(unittest.TestCase):
prompt_token_ids=[1, 2, 3],
mm_features=[mm_feature],
)
session.append_output_token_ids(10)
session.num_computed_tokens = len(session.prompt_token_ids)
mm_feature = MultiModalFeatureSpec(
@@ -316,6 +321,7 @@ class TestStreamingScheduler(unittest.TestCase):
)
scheduler.add_request(session)
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
session.append_output_token_ids(10)
session.num_computed_tokens = len(session.prompt_token_ids)
next_request = DummyRequest(
@@ -716,3 +722,54 @@ def test_next_chunk_fully_computed_after_stop_on_accepted_draft(async_scheduling
assert list(session.prompt_token_ids) == [1, 2, 3, 10, 7, 8]
assert output.num_scheduled_tokens["session"] == 2
@pytest.mark.parametrize("chunk_queued_before_stop", [False, True])
def test_session_resumes_after_stop_while_preempted(chunk_queued_before_stop):
"""A session preempted with a step in flight can stop when that step's
output is delivered. It must stay schedulable and keep its tokens."""
scheduler = create_async_scheduler(async_scheduling=True)
session = DummyRequest("session", prompt_token_ids=list(range(10)))
scheduler.add_request(session)
scheduler.update_from_output(scheduler.schedule(), _model_output("session", [10]))
in_flight = scheduler.schedule()
next_chunk = DummyRequest("session", prompt_token_ids=[20, 21])
if chunk_queued_before_stop:
scheduler.add_request(next_chunk)
scheduler.running.remove(session)
scheduler._preempt_request(session, timestamp=0.0)
scheduler.update_from_output(in_flight, _model_output("session", [STOP_TOKEN]))
# The parked variant holds a streaming-input slot until the next chunk.
assert scheduler.num_waiting_for_streaming_input == (
0 if chunk_queued_before_stop else 1
)
if not chunk_queued_before_stop:
scheduler.add_request(next_chunk)
output = scheduler.schedule()
assert scheduler.num_waiting_for_streaming_input == 0
expected = list(range(10)) + [10, 20, 21]
assert list(session.all_token_ids) == expected
assert list(session.prompt_token_ids) == expected
assert output.num_scheduled_tokens["session"] == len(expected)
def test_non_resumable_request_stays_dequeued_after_stop_while_preempted():
"""Only resumable requests continue after a stop; a request that finishes
while preempted must not be put back in a queue."""
scheduler = create_async_scheduler(async_scheduling=True)
request = DummyRequest("req", resumable=False, prompt_token_ids=list(range(10)))
scheduler.add_request(request)
scheduler.update_from_output(scheduler.schedule(), _model_output("req", [10]))
in_flight = scheduler.schedule()
scheduler.running.remove(request)
scheduler._preempt_request(request, timestamp=0.0)
scheduler.update_from_output(in_flight, _model_output("req", [STOP_TOKEN]))
assert request.status == RequestStatus.FINISHED_STOPPED
assert request not in scheduler.waiting
assert request not in scheduler.kv_holding_waiting
assert request not in scheduler.deferred_waiting
assert "req" not in scheduler.schedule().num_scheduled_tokens
+24
View File
@@ -92,6 +92,7 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
layers[name] = SimpleNamespace(
get_kv_cache_spec=lambda _, spec=specs[name]: spec,
get_attn_backend=lambda backend=backend: backend,
is_draft_layer=False,
)
monkeypatch.setattr(attn_utils, "get_layers_from_vllm_config", lambda *_: layers)
monkeypatch.setattr(
@@ -112,6 +113,29 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
assert all(spec.block_size == block_size for spec in specs.values())
def test_get_kv_cache_spec_tags_draft_layers(monkeypatch):
spec = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=128, dtype=torch.bfloat16
)
backend = SimpleNamespace(
customize_spec=AttentionBackend.customize_spec,
get_supported_kernel_block_sizes=lambda _=None: [16],
)
layers = {
name: SimpleNamespace(
get_kv_cache_spec=lambda _: spec,
get_attn_backend=lambda: backend,
is_draft_layer=name == "draft",
)
for name in ("target", "draft")
}
monkeypatch.setattr(attn_utils, "get_layers_from_vllm_config", lambda *_: layers)
config = SimpleNamespace(attention_config=SimpleNamespace(hisparse_config=None))
specs = attn_utils.get_kv_cache_spec(config)
assert specs["draft"].is_draft and not specs["target"].is_draft
class _FakeMetadataBuilder:
def __init__(self, support: AttentionCGSupport, varlen_bound: int | None = None):
self.support = support
+101
View File
@@ -305,3 +305,104 @@ def test_dummy_request_slot_mapping_is_pad():
num_tokens_padded=3,
)
assert dummy[0].tolist() == [PAD_SLOT_ID] * 3
@pytest.fixture
def populated_block_tables():
tables = BlockTables([16, 32], 8, 64, [3, 5], torch.device("cuda"))
for req_index in range(8):
tables.append_block_ids(
req_index, ([req_index + 1] * 3, [req_index + 1] * 5), overwrite=True
)
tables.apply_staged_writes()
tables.gather_block_tables(torch.arange(8, device="cuda"), num_reqs_padded=8)
return tables
@pytest.mark.parametrize("previous_dummy", [False, True])
@pytest.mark.parametrize("next_dummy", [False, True])
def test_shared_block_tables_clear_rows_from_larger_batch(
populated_block_tables, previous_dummy, next_dummy
):
"""A smaller target batch must leave no stale rows for larger draft graphs."""
tables = populated_block_tables
if previous_dummy:
for table in tables.get_dummy_block_tables(8):
table.fill_(99) # Profiling supplies valid context/state slots.
ptrs = [table.data_ptr() for table in tables.input_block_tables]
idx = torch.tensor([7, 2, 0, 5, 1], device="cuda")
for num_reqs in (3, 5, 0):
if next_dummy:
out = tables.get_dummy_block_tables(num_reqs)
num_rows = num_reqs
else:
num_rows = num_reqs + 1
out = tables.gather_block_tables(idx[:num_reqs], num_rows)
for table, view, ptr in zip(tables.input_block_tables, out, ptrs):
assert view.shape == (num_rows, table.shape[1])
assert table.data_ptr() == ptr
if num_rows:
assert view.data_ptr() == ptr
assert torch.count_nonzero(table[num_reqs:]).item() == 0
if next_dummy:
assert torch.count_nonzero(view).item() == 0
view.fill_(99)
else:
expected = (idx[:num_reqs, None] + 1).expand(-1, table.shape[1])
torch.testing.assert_close(view[:num_reqs], expected.to(torch.int32))
def test_external_block_table_output_preserves_shared_rows(populated_block_tables):
"""PCP-local gathers must neither clear shared rows nor reset their tracking."""
tables = populated_block_tables
previous = [table.clone() for table in tables.input_block_tables]
out = tuple(torch.full_like(table[:3], -1) for table in previous)
out_ptrs = torch.tensor(
[table.data_ptr() for table in out], dtype=torch.uint64, device="cuda"
)
idx = torch.tensor([7, 2], device="cuda")
result = tables.gather_block_tables(idx, 3, out=out, out_ptrs=out_ptrs)
for table, before, view in zip(tables.input_block_tables, previous, result):
torch.testing.assert_close(table, before)
assert view[:, 0].tolist() == [8, 3, 0]
tables.gather_block_tables(idx[:1], num_reqs_padded=1)
for table in tables.input_block_tables:
assert torch.count_nonzero(table[1:]).item() == 0
@pytest.mark.parametrize("use_cuda_graph", [False, True])
def test_larger_draft_batch_uses_null_kpool_slots(
populated_block_tables, use_cuda_graph
):
"""Draft padding must not resolve to another request's physical KPool slots."""
from vllm.v1.attention.backends.mla.indexer import compute_kpool_tail_slot_mapping
tables = populated_block_tables
idx = torch.tensor([2, 0, 4, 1, 3], device="cuda")
query_start_loc = torch.arange(9, dtype=torch.int32, device="cuda").clamp_max_(5)
slot_mapping = torch.full((8,), -1, dtype=torch.int64, device="cuda")
positions = torch.arange(8, dtype=torch.int64, device="cuda")
def compute_slots():
return compute_kpool_tail_slot_mapping(
slot_mapping,
tables.input_block_tables[0],
query_start_loc,
positions,
num_actual_tokens=8,
num_reqs=8,
kpool=8,
)
if use_cuda_graph:
compute_slots()
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
slots = compute_slots()
tables.gather_block_tables(idx, num_reqs_padded=5)
if use_cuda_graph:
graph.replay()
else:
slots = compute_slots()
assert slots.tolist() == [24, 9, 42, 19, 36, -1, -1, -1]
+25 -3
View File
@@ -22,7 +22,12 @@ def _make_connector(
backend.bind_connector_metadata.side_effect = lambda _: events.append("bind")
backend.start_load_kv.side_effect = lambda *_args, **_kwargs: events.append("start")
backend.wait_for_save.side_effect = lambda: events.append("wait")
backend.get_transfer_results.return_value = KVConnectorTransferResults()
def get_transfer_results(_finished_req_ids):
events.append("results")
return KVConnectorTransferResults()
backend.get_transfer_results.side_effect = get_transfer_results
backend.get_block_ids_with_load_errors.return_value = set()
backend.get_kv_connector_stats.return_value = None
backend.get_kv_connector_kv_cache_events.return_value = None
@@ -73,7 +78,7 @@ def test_load_start_phase(
)
connector.post_forward(set())
assert events == ["handle", "bind", "start", "wait", "clear"]
assert events == ["handle", "bind", "start", "wait", "results", "clear"]
kwargs = connector.kv_connector.start_load_kv.call_args.kwargs
assert kwargs["request_state_indices"] is request_indices
@@ -94,4 +99,21 @@ def test_no_forward_starts_deferred_load_once(monkeypatch: pytest.MonkeyPatch):
connector.no_forward(_scheduler_output(False))
assert events == ["handle", "bind", "start", "wait", "clear"]
assert events == ["handle", "bind", "start", "wait", "results", "clear"]
def test_deferred_finalize_collects_results_before_saving(
monkeypatch: pytest.MonkeyPatch,
):
"""With a drafter, the transfer results are collected right after the
target forward, and the saves and metadata are finalized after the
drafter writes its KV."""
events: list[str] = []
connector = _make_connector(monkeypatch, events)
connector.pre_forward(_scheduler_output(False))
connector.post_forward(set(), defer_finalize=True)
assert events == ["handle", "bind", "start", "results"]
connector.finalize_step()
assert events == ["handle", "bind", "start", "results", "wait", "clear"]
@@ -88,6 +88,7 @@ def _make_runner(
"""Stub model runner exposing only what the warmup entry points read."""
return SimpleNamespace(
num_speculative_steps=num_spec_steps,
speculative_config=SimpleNamespace() if num_spec_steps else None,
adaptive_verification=None,
rejection_sampler=None,
decode_query_len=num_spec_steps + 1,
+25
View File
@@ -74,6 +74,7 @@ def test_kernel_warmup_restores_uncalibrated_adaptive_manager(monkeypatch, fail_
runner = SimpleNamespace(
adaptive_verification=manager,
rejection_sampler=rejection_sampler,
speculative_config=SimpleNamespace(),
)
def run_steps(model_runner, execute, sample, greedy=False):
@@ -91,3 +92,27 @@ def test_kernel_warmup_restores_uncalibrated_adaptive_manager(monkeypatch, fail_
assert runner.adaptive_verification is manager
assert manager.cost_tables is None
assert rejection_sampler.enable_adaptive_verification
@pytest.mark.parametrize("speculative", [False, True])
def test_kernel_warmup_runs_same_rounds_on_every_pp_stage(monkeypatch, speculative):
"""PP receivers need a matching sender for every speculative warmup round."""
rounds = []
monkeypatch.setattr(
warmup,
"_warmup_kernels",
lambda runner, execute, sample, greedy=False: rounds.append(greedy),
)
for is_last_pp_rank in (False, True):
rounds.clear()
runner = SimpleNamespace(
adaptive_verification=None,
speculative_config=SimpleNamespace() if speculative else None,
rejection_sampler=(
SimpleNamespace(enable_adaptive_verification=False)
if is_last_pp_rank and speculative
else None
),
)
warmup.warmup_kernels(runner, _fail, _fail)
assert rounds == ([False, True] if speculative else [False])