Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-15 21:16:39 +07:00
1693 changed files with 99919 additions and 32440 deletions
@@ -35,6 +35,7 @@ def _run_prepare(
input_buffers = SimpleNamespace(
input_ids=torch.full((max_num_tokens,), -1, dtype=torch.int32, device=device),
positions=torch.full((max_num_tokens,), -1, dtype=torch.int64, device=device),
is_padding=torch.zeros(max_num_tokens, dtype=torch.bool, device=device),
query_start_loc=torch.full(
(max_num_reqs + 1,), -1, dtype=torch.int32, device=device
),
@@ -102,7 +103,7 @@ def _run_prepare(
block_tables=SimpleNamespace(
slot_mappings=query_slot_mapping.unsqueeze(0),
input_block_tables=[block_table],
kernel_block_sizes=[4],
block_sizes=[4],
cp_rank=cp_rank,
cp_size=cp_size,
cp_interleave=cp_interleave,
@@ -177,6 +178,11 @@ def test_prepare_dflash_inputs_excludes_rejected_context_suffix(cp_rank, cp_size
assert out.sample_idx_mapping[:3].tolist() == [2, 2, 2]
assert out.temperature[2].item() == 1.0
assert out.seeds[2].item() == 17
assert not out.input_buffers.is_padding[:3].any()
assert out.input_buffers.is_padding[3:].all()
assert (out.input_buffers.input_ids[3:] == 0).all()
assert (out.input_buffers.positions[3:] == 0).all()
assert (out.query_slot_mapping[3:] == PAD_SLOT_ID).all()
def test_prepare_dflash_inputs_excludes_rejected_context_suffix_with_dcp():
+6
View File
@@ -963,6 +963,9 @@ def test_propose(method, attn_backend, num_speculative_tokens, monkeypatch):
# Mock draft_attn_groups for attention metadata building.
mock_attn_group = mock.MagicMock()
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
mock_attn_group.build_metadata_for_drafting = (
attn_metadata_builder.build_for_drafting
)
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
mock_attn_group.kv_cache_spec = attn_metadata_builder.kv_cache_spec
proposer.draft_attn_groups = [mock_attn_group]
@@ -1066,6 +1069,9 @@ def test_propose_stores_probabilistic_draft_probs(attn_backend, monkeypatch):
)
mock_attn_group = mock.MagicMock()
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
mock_attn_group.build_metadata_for_drafting = (
attn_metadata_builder.build_for_drafting
)
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
mock_attn_group.kv_cache_spec = attn_metadata_builder.kv_cache_spec
proposer.draft_attn_groups = [mock_attn_group]
+35 -4
View File
@@ -290,18 +290,49 @@ def test_context_anchor_is_last_committed_normalized_feature():
anchor_hidden=torch.full((4, 4), 123.0),
anchor_valid=torch.ones(4, dtype=torch.bool),
)
batch = SimpleNamespace(num_reqs=2, query_start_loc=torch.tensor([0, 2, 6]))
LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([0, 2]))
LiLiCorrSpeculator.prepare_context_anchor(
spec, 2, torch.tensor([0, 2, 6]), torch.tensor([0, 2])
)
torch.testing.assert_close(spec.anchor_hidden[:2], norm(hidden[[1, 3]]))
assert spec.anchor_valid.tolist() == [True, True, False, False]
assert not spec.anchor_hidden[2:].any()
# A smaller reordered batch must not retain an old request's anchor.
batch = SimpleNamespace(num_reqs=1, query_start_loc=torch.tensor([0, 3]))
LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([1]))
LiLiCorrSpeculator.prepare_context_anchor(
spec, 1, torch.tensor([0, 3]), torch.tensor([1])
)
torch.testing.assert_close(spec.anchor_hidden[0], norm(hidden[1]))
assert not spec.anchor_hidden[1:].any()
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_captured_context_anchor_rereads_fixed_inputs():
"""The anchor runs in the draft's FULL graph: each replay must re-read its
fixed-address inputs and ignore a padded request's stale rejected count."""
dev = torch.device("cuda")
query_start_loc = torch.zeros(3, dtype=torch.int32, device=dev)
num_rejected = torch.zeros(2, dtype=torch.int32, device=dev)
spec = SimpleNamespace(
hidden_states=torch.randn(8, 2, device=dev),
model=SimpleNamespace(model=SimpleNamespace(hidden_norm=lambda x: 2 * x)),
anchor_hidden=torch.zeros(2, 2, device=dev),
anchor_valid=torch.zeros(2, dtype=torch.bool, device=dev),
)
args = (spec, 2, query_start_loc, num_rejected)
LiLiCorrSpeculator.prepare_context_anchor(*args)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
LiLiCorrSpeculator.prepare_context_anchor(*args)
# One live request padded to two; the second case anchors on a prefill row
# past the graph's context rows, stored eagerly before replay.
for loc, rejected, row in (([0, 2, 2], [1, 5], 0), ([0, 7, 7], [0, 0], 6)):
query_start_loc.copy_(torch.tensor(loc))
num_rejected.copy_(torch.tensor(rejected))
graph.replay()
torch.testing.assert_close(spec.anchor_hidden[0], 2 * spec.hidden_states[row])
assert spec.anchor_valid.tolist() == [True, False]
assert not spec.anchor_hidden[1].any()
@pytest.mark.parametrize("lilicorr", [False, True])
def test_candidate_generation_routes_scores_and_adaptive_inputs(monkeypatch, lilicorr):
"""Both heads share the walk, including a padded request and adaptive scores."""
+10 -14
View File
@@ -2,10 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for SpecDecodeBaseProposer.initialize_attn_backend.
Block tables are stored at kernel-block granularity, so the proposer's
``block_size`` (used for slot-mapping math) must be the kernel block size,
not the KV cache manager's block size — the two differ when manager blocks
are split for the attention kernel. The value must also be deterministic:
Block tables are stored at manager-block granularity, so the proposer's
``block_size`` (used for slot-mapping math) must be the KV cache manager's
block size. Attention groups map to kernel blocks for their metadata builders.
The value must also be deterministic:
``_draft_attn_layer_names`` is a set, whose iteration order varies across
processes, so anything derived from iteration order must not leak into
``block_size``.
@@ -68,9 +68,9 @@ def _make_kv_cache_config(layer_names: set[str]) -> KVCacheConfig:
return SimpleNamespace(kv_cache_groups=[group]) # type: ignore[return-value]
def test_block_size_uses_kernel_block_size(monkeypatch: pytest.MonkeyPatch):
"""The proposer's slot-mapping math runs against the kernel-granularity
block table, so block_size must come from kernel_block_sizes."""
def test_block_size_uses_manager_block_size(monkeypatch: pytest.MonkeyPatch):
"""The proposer's slot-mapping math runs against the manager-granularity
block table; attention groups map kernel blocks for metadata builders."""
layer_names = {"draft.0.self_attn.attn"}
proposer = _make_proposer(monkeypatch, layer_names)
@@ -79,12 +79,8 @@ def test_block_size_uses_kernel_block_size(monkeypatch: pytest.MonkeyPatch):
kernel_block_sizes=[KERNEL_BLOCK_SIZE],
)
assert proposer.block_size == KERNEL_BLOCK_SIZE
assert proposer.block_size != SCHEDULER_BLOCK_SIZE
# The metadata builder keeps receiving the kernel block size as well.
group = proposer.draft_attn_groups[0]
assert isinstance(group, _FakeAttentionGroup)
assert group.kernel_block_size == KERNEL_BLOCK_SIZE
assert proposer.block_size == SCHEDULER_BLOCK_SIZE
assert proposer.draft_attn_groups[0].kernel_block_size == KERNEL_BLOCK_SIZE
def test_block_size_falls_back_to_kv_cache_spec(monkeypatch: pytest.MonkeyPatch):
@@ -112,4 +108,4 @@ def test_draft_layer_iteration_is_deterministic(monkeypatch: pytest.MonkeyPatch)
)
assert len(proposer.draft_attn_groups) == 1
assert proposer.draft_attn_groups[0].layer_names == expected_order
assert proposer.block_size == KERNEL_BLOCK_SIZE
assert proposer.block_size == SCHEDULER_BLOCK_SIZE
+1 -1
View File
@@ -147,7 +147,7 @@ def _propose(
next_prefill_tokens=torch.zeros(B, dtype=torch.int32, device=DEVICE),
temperature=torch.zeros(B, dtype=torch.float32, device=DEVICE),
seeds=torch.zeros(B, dtype=torch.int64, device=DEVICE),
dp_sync=None,
dp_sync_state=None,
)
return drafts.cpu().tolist()