Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -35,6 +35,7 @@ def _run_prepare(
|
||||
input_buffers = SimpleNamespace(
|
||||
input_ids=torch.full((max_num_tokens,), -1, dtype=torch.int32, device=device),
|
||||
positions=torch.full((max_num_tokens,), -1, dtype=torch.int64, device=device),
|
||||
is_padding=torch.zeros(max_num_tokens, dtype=torch.bool, device=device),
|
||||
query_start_loc=torch.full(
|
||||
(max_num_reqs + 1,), -1, dtype=torch.int32, device=device
|
||||
),
|
||||
@@ -102,7 +103,7 @@ def _run_prepare(
|
||||
block_tables=SimpleNamespace(
|
||||
slot_mappings=query_slot_mapping.unsqueeze(0),
|
||||
input_block_tables=[block_table],
|
||||
kernel_block_sizes=[4],
|
||||
block_sizes=[4],
|
||||
cp_rank=cp_rank,
|
||||
cp_size=cp_size,
|
||||
cp_interleave=cp_interleave,
|
||||
@@ -177,6 +178,11 @@ def test_prepare_dflash_inputs_excludes_rejected_context_suffix(cp_rank, cp_size
|
||||
assert out.sample_idx_mapping[:3].tolist() == [2, 2, 2]
|
||||
assert out.temperature[2].item() == 1.0
|
||||
assert out.seeds[2].item() == 17
|
||||
assert not out.input_buffers.is_padding[:3].any()
|
||||
assert out.input_buffers.is_padding[3:].all()
|
||||
assert (out.input_buffers.input_ids[3:] == 0).all()
|
||||
assert (out.input_buffers.positions[3:] == 0).all()
|
||||
assert (out.query_slot_mapping[3:] == PAD_SLOT_ID).all()
|
||||
|
||||
|
||||
def test_prepare_dflash_inputs_excludes_rejected_context_suffix_with_dcp():
|
||||
|
||||
@@ -963,6 +963,9 @@ def test_propose(method, attn_backend, num_speculative_tokens, monkeypatch):
|
||||
# Mock draft_attn_groups for attention metadata building.
|
||||
mock_attn_group = mock.MagicMock()
|
||||
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
|
||||
mock_attn_group.build_metadata_for_drafting = (
|
||||
attn_metadata_builder.build_for_drafting
|
||||
)
|
||||
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
|
||||
mock_attn_group.kv_cache_spec = attn_metadata_builder.kv_cache_spec
|
||||
proposer.draft_attn_groups = [mock_attn_group]
|
||||
@@ -1066,6 +1069,9 @@ def test_propose_stores_probabilistic_draft_probs(attn_backend, monkeypatch):
|
||||
)
|
||||
mock_attn_group = mock.MagicMock()
|
||||
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
|
||||
mock_attn_group.build_metadata_for_drafting = (
|
||||
attn_metadata_builder.build_for_drafting
|
||||
)
|
||||
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
|
||||
mock_attn_group.kv_cache_spec = attn_metadata_builder.kv_cache_spec
|
||||
proposer.draft_attn_groups = [mock_attn_group]
|
||||
|
||||
@@ -290,18 +290,49 @@ def test_context_anchor_is_last_committed_normalized_feature():
|
||||
anchor_hidden=torch.full((4, 4), 123.0),
|
||||
anchor_valid=torch.ones(4, dtype=torch.bool),
|
||||
)
|
||||
batch = SimpleNamespace(num_reqs=2, query_start_loc=torch.tensor([0, 2, 6]))
|
||||
LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([0, 2]))
|
||||
LiLiCorrSpeculator.prepare_context_anchor(
|
||||
spec, 2, torch.tensor([0, 2, 6]), torch.tensor([0, 2])
|
||||
)
|
||||
torch.testing.assert_close(spec.anchor_hidden[:2], norm(hidden[[1, 3]]))
|
||||
assert spec.anchor_valid.tolist() == [True, True, False, False]
|
||||
assert not spec.anchor_hidden[2:].any()
|
||||
# A smaller reordered batch must not retain an old request's anchor.
|
||||
batch = SimpleNamespace(num_reqs=1, query_start_loc=torch.tensor([0, 3]))
|
||||
LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([1]))
|
||||
LiLiCorrSpeculator.prepare_context_anchor(
|
||||
spec, 1, torch.tensor([0, 3]), torch.tensor([1])
|
||||
)
|
||||
torch.testing.assert_close(spec.anchor_hidden[0], norm(hidden[1]))
|
||||
assert not spec.anchor_hidden[1:].any()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_captured_context_anchor_rereads_fixed_inputs():
|
||||
"""The anchor runs in the draft's FULL graph: each replay must re-read its
|
||||
fixed-address inputs and ignore a padded request's stale rejected count."""
|
||||
dev = torch.device("cuda")
|
||||
query_start_loc = torch.zeros(3, dtype=torch.int32, device=dev)
|
||||
num_rejected = torch.zeros(2, dtype=torch.int32, device=dev)
|
||||
spec = SimpleNamespace(
|
||||
hidden_states=torch.randn(8, 2, device=dev),
|
||||
model=SimpleNamespace(model=SimpleNamespace(hidden_norm=lambda x: 2 * x)),
|
||||
anchor_hidden=torch.zeros(2, 2, device=dev),
|
||||
anchor_valid=torch.zeros(2, dtype=torch.bool, device=dev),
|
||||
)
|
||||
args = (spec, 2, query_start_loc, num_rejected)
|
||||
LiLiCorrSpeculator.prepare_context_anchor(*args)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
LiLiCorrSpeculator.prepare_context_anchor(*args)
|
||||
# One live request padded to two; the second case anchors on a prefill row
|
||||
# past the graph's context rows, stored eagerly before replay.
|
||||
for loc, rejected, row in (([0, 2, 2], [1, 5], 0), ([0, 7, 7], [0, 0], 6)):
|
||||
query_start_loc.copy_(torch.tensor(loc))
|
||||
num_rejected.copy_(torch.tensor(rejected))
|
||||
graph.replay()
|
||||
torch.testing.assert_close(spec.anchor_hidden[0], 2 * spec.hidden_states[row])
|
||||
assert spec.anchor_valid.tolist() == [True, False]
|
||||
assert not spec.anchor_hidden[1].any()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("lilicorr", [False, True])
|
||||
def test_candidate_generation_routes_scores_and_adaptive_inputs(monkeypatch, lilicorr):
|
||||
"""Both heads share the walk, including a padded request and adaptive scores."""
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for SpecDecodeBaseProposer.initialize_attn_backend.
|
||||
|
||||
Block tables are stored at kernel-block granularity, so the proposer's
|
||||
``block_size`` (used for slot-mapping math) must be the kernel block size,
|
||||
not the KV cache manager's block size — the two differ when manager blocks
|
||||
are split for the attention kernel. The value must also be deterministic:
|
||||
Block tables are stored at manager-block granularity, so the proposer's
|
||||
``block_size`` (used for slot-mapping math) must be the KV cache manager's
|
||||
block size. Attention groups map to kernel blocks for their metadata builders.
|
||||
The value must also be deterministic:
|
||||
``_draft_attn_layer_names`` is a set, whose iteration order varies across
|
||||
processes, so anything derived from iteration order must not leak into
|
||||
``block_size``.
|
||||
@@ -68,9 +68,9 @@ def _make_kv_cache_config(layer_names: set[str]) -> KVCacheConfig:
|
||||
return SimpleNamespace(kv_cache_groups=[group]) # type: ignore[return-value]
|
||||
|
||||
|
||||
def test_block_size_uses_kernel_block_size(monkeypatch: pytest.MonkeyPatch):
|
||||
"""The proposer's slot-mapping math runs against the kernel-granularity
|
||||
block table, so block_size must come from kernel_block_sizes."""
|
||||
def test_block_size_uses_manager_block_size(monkeypatch: pytest.MonkeyPatch):
|
||||
"""The proposer's slot-mapping math runs against the manager-granularity
|
||||
block table; attention groups map kernel blocks for metadata builders."""
|
||||
layer_names = {"draft.0.self_attn.attn"}
|
||||
proposer = _make_proposer(monkeypatch, layer_names)
|
||||
|
||||
@@ -79,12 +79,8 @@ def test_block_size_uses_kernel_block_size(monkeypatch: pytest.MonkeyPatch):
|
||||
kernel_block_sizes=[KERNEL_BLOCK_SIZE],
|
||||
)
|
||||
|
||||
assert proposer.block_size == KERNEL_BLOCK_SIZE
|
||||
assert proposer.block_size != SCHEDULER_BLOCK_SIZE
|
||||
# The metadata builder keeps receiving the kernel block size as well.
|
||||
group = proposer.draft_attn_groups[0]
|
||||
assert isinstance(group, _FakeAttentionGroup)
|
||||
assert group.kernel_block_size == KERNEL_BLOCK_SIZE
|
||||
assert proposer.block_size == SCHEDULER_BLOCK_SIZE
|
||||
assert proposer.draft_attn_groups[0].kernel_block_size == KERNEL_BLOCK_SIZE
|
||||
|
||||
|
||||
def test_block_size_falls_back_to_kv_cache_spec(monkeypatch: pytest.MonkeyPatch):
|
||||
@@ -112,4 +108,4 @@ def test_draft_layer_iteration_is_deterministic(monkeypatch: pytest.MonkeyPatch)
|
||||
)
|
||||
assert len(proposer.draft_attn_groups) == 1
|
||||
assert proposer.draft_attn_groups[0].layer_names == expected_order
|
||||
assert proposer.block_size == KERNEL_BLOCK_SIZE
|
||||
assert proposer.block_size == SCHEDULER_BLOCK_SIZE
|
||||
|
||||
@@ -147,7 +147,7 @@ def _propose(
|
||||
next_prefill_tokens=torch.zeros(B, dtype=torch.int32, device=DEVICE),
|
||||
temperature=torch.zeros(B, dtype=torch.float32, device=DEVICE),
|
||||
seeds=torch.zeros(B, dtype=torch.int64, device=DEVICE),
|
||||
dp_sync=None,
|
||||
dp_sync_state=None,
|
||||
)
|
||||
return drafts.cpu().tolist()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user