[MRV2] Miscellaneous code structure updates (#60754)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -147,7 +147,7 @@ def _propose(
|
||||
next_prefill_tokens=torch.zeros(B, dtype=torch.int32, device=DEVICE),
|
||||
temperature=torch.zeros(B, dtype=torch.float32, device=DEVICE),
|
||||
seeds=torch.zeros(B, dtype=torch.int64, device=DEVICE),
|
||||
dp_sync=None,
|
||||
dp_sync_state=None,
|
||||
)
|
||||
return drafts.cpu().tolist()
|
||||
|
||||
|
||||
@@ -63,13 +63,11 @@ def test_load_start_phase(
|
||||
|
||||
request_indices = torch.tensor([3, 1])
|
||||
request_ids = ["first", "second"]
|
||||
attn_metadata = {"layer": object()}
|
||||
connector.pre_forward(
|
||||
output,
|
||||
request_state_indices=request_indices,
|
||||
request_ids=request_ids,
|
||||
attn_metadata=attn_metadata,
|
||||
input_batch = SimpleNamespace(
|
||||
idx_mapping=request_indices, req_ids=request_ids, num_tokens=5
|
||||
)
|
||||
attn_metadata = {"layer": object()}
|
||||
connector.pre_forward(output, input_batch, attn_metadata=attn_metadata)
|
||||
assert events == (
|
||||
["handle", "bind", "start"] if has_sync_kv_loads else ["handle", "bind"]
|
||||
)
|
||||
@@ -80,6 +78,8 @@ def test_load_start_phase(
|
||||
kwargs = connector.kv_connector.start_load_kv.call_args.kwargs
|
||||
assert kwargs["request_state_indices"] is request_indices
|
||||
assert kwargs["request_ids"] is request_ids
|
||||
assert kwargs["num_tokens"] == 5
|
||||
assert kwargs["scheduler_output"] is output
|
||||
assert kwargs["attn_metadata"] is attn_metadata
|
||||
|
||||
# A subsequent step without a forward must not reuse the prior batch.
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
from vllm.v1.worker.gpu.async_utils import async_copy_to_np
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.model_runner import ExecuteModelState, GPUModelRunner
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import get_drafter_hidden_states
|
||||
|
||||
|
||||
def test_non_last_pp_rank_uses_global_batch_for_sample_feedback():
|
||||
@@ -43,7 +44,7 @@ def test_non_last_pp_rank_uses_global_batch_for_sample_feedback():
|
||||
slot_mappings_by_layer=None,
|
||||
hidden_states=None,
|
||||
aux_hidden_states=None,
|
||||
dp_sync=None,
|
||||
dp_sync_state=None,
|
||||
finished_req_ids=set(),
|
||||
ec_connector_output=None,
|
||||
cudagraph_stats=None,
|
||||
@@ -342,16 +343,15 @@ def test_capture_model_profile_only_skips_lock(monkeypatch):
|
||||
@pytest.mark.parametrize("target_buffer", ["absent", "none", "tensor"])
|
||||
def test_get_drafter_hidden_states_tolerates_missing_target_buffer(target_buffer):
|
||||
"""Targets allocate the MTP hidden buffer only for hidden-state drafters."""
|
||||
runner = GPUModelRunner.__new__(GPUModelRunner)
|
||||
hidden_states = torch.zeros(4, 8)
|
||||
buffer = torch.arange(16 * 8, dtype=torch.float32).view(16, 8)
|
||||
if target_buffer == "absent":
|
||||
runner.model = SimpleNamespace()
|
||||
model = SimpleNamespace()
|
||||
else:
|
||||
returned = buffer if target_buffer == "tensor" else None
|
||||
runner.model = SimpleNamespace(get_mtp_target_hidden_states=lambda: returned)
|
||||
model = SimpleNamespace(get_mtp_target_hidden_states=lambda: returned)
|
||||
|
||||
out = runner._get_drafter_hidden_states(hidden_states)
|
||||
out = get_drafter_hidden_states(model, hidden_states)
|
||||
|
||||
if target_buffer == "tensor":
|
||||
assert torch.equal(out, buffer[:4])
|
||||
|
||||
@@ -185,7 +185,7 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
|
||||
slot_mappings_by_layer=None,
|
||||
hidden_states=None,
|
||||
aux_hidden_states=None,
|
||||
dp_sync=None,
|
||||
dp_sync_state=None,
|
||||
finished_req_ids=set(),
|
||||
ec_connector_output=None,
|
||||
routed_experts=None,
|
||||
|
||||
@@ -350,10 +350,10 @@ def test_decode_threshold_only_applies_when_every_rank_is_a_uniform_decode():
|
||||
|
||||
def test_microbatching_pads_all_ranks_to_the_largest():
|
||||
"""Ranks must run the same token count so microbatch sizes line up."""
|
||||
desc, dp_sync = _sync_dp([200, 256])
|
||||
desc, dp_sync_state = _sync_dp([200, 256])
|
||||
assert desc.num_tokens == 256
|
||||
assert dp_sync is not None
|
||||
assert dp_sync.num_tokens_across_dp.tolist() == [256, 256]
|
||||
assert dp_sync_state is not None
|
||||
assert dp_sync_state.num_tokens_across_dp.tolist() == [256, 256]
|
||||
|
||||
|
||||
def test_microbatching_survives_a_rank_that_cannot_fill_it():
|
||||
@@ -959,7 +959,7 @@ def test_microbatched_graph_needs_every_rank_to_reach_the_split():
|
||||
assert desc.num_ubatches == 2
|
||||
|
||||
# Rank 1 pads the group up to a 128-token graph, but rank 0 only reaches 33.
|
||||
desc, dp_sync = _sync_dp([33, 128], uniform, cudagraph_manager=manager)
|
||||
desc, dp_sync_state = _sync_dp([33, 128], uniform, cudagraph_manager=manager)
|
||||
assert desc.cg_mode == CUDAGraphMode.NONE
|
||||
assert desc.num_ubatches == 2
|
||||
assert dp_sync is not None and dp_sync.eager
|
||||
assert dp_sync_state is not None and dp_sync_state.eager
|
||||
|
||||
Reference in New Issue
Block a user