[ModelRunnerV2] Avoid pipeline parallel bubbles (#42187)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -167,7 +167,9 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
|
||||
events = []
|
||||
runner = _make_runner(is_last_pp_rank=False, num_speculative_steps=0)
|
||||
runner.execute_model_state = SimpleNamespace(
|
||||
input_batch=SimpleNamespace(num_reqs=2),
|
||||
input_batch=SimpleNamespace(
|
||||
num_reqs=2, idx_mapping=torch.zeros(2, dtype=torch.int32)
|
||||
),
|
||||
attn_metadata=None,
|
||||
slot_mappings_by_layer=None,
|
||||
hidden_states=None,
|
||||
@@ -175,18 +177,19 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
|
||||
finished_req_ids=set(),
|
||||
num_tokens_across_dp=None,
|
||||
)
|
||||
runner.postprocess = lambda *args, **kwargs: events.append("postprocess")
|
||||
runner.eplb.step = lambda *args, **kwargs: events.append("eplb")
|
||||
monkeypatch.setattr(
|
||||
mrv2,
|
||||
"pp_receive",
|
||||
lambda *args, **kwargs: (
|
||||
torch.zeros((2, 1), dtype=torch.long),
|
||||
torch.ones(2, dtype=torch.int32),
|
||||
torch.zeros(2, dtype=torch.int32),
|
||||
),
|
||||
runner.req_states = SimpleNamespace()
|
||||
|
||||
def fake_receive(*args, **kwargs):
|
||||
events.append("receive")
|
||||
# all_decode_next=True, so model_state.postprocess_state is skipped.
|
||||
return True
|
||||
|
||||
runner.pp_handler = SimpleNamespace(receive=fake_receive)
|
||||
runner.postprocess_num_computed_tokens = lambda *args, **kwargs: events.append(
|
||||
"postprocess_num_computed_tokens"
|
||||
)
|
||||
runner.eplb.step = lambda *args, **kwargs: events.append("eplb")
|
||||
|
||||
output = mrv2.GPUModelRunner.sample_tokens(runner, None)
|
||||
assert output in (EMPTY_MODEL_RUNNER_OUTPUT, None)
|
||||
assert events == ["postprocess", "eplb"]
|
||||
assert events == ["receive", "postprocess_num_computed_tokens", "eplb"]
|
||||
|
||||
Reference in New Issue
Block a user