[ModelRunnerV2] Avoid pipeline parallel bubbles (#42187)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-06-02 14:02:01 -07:00
committed by GitHub
parent 557781131a
commit e15f20258b
19 changed files with 615 additions and 144 deletions
@@ -167,7 +167,9 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
events = []
runner = _make_runner(is_last_pp_rank=False, num_speculative_steps=0)
runner.execute_model_state = SimpleNamespace(
input_batch=SimpleNamespace(num_reqs=2),
input_batch=SimpleNamespace(
num_reqs=2, idx_mapping=torch.zeros(2, dtype=torch.int32)
),
attn_metadata=None,
slot_mappings_by_layer=None,
hidden_states=None,
@@ -175,18 +177,19 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
finished_req_ids=set(),
num_tokens_across_dp=None,
)
runner.postprocess = lambda *args, **kwargs: events.append("postprocess")
runner.eplb.step = lambda *args, **kwargs: events.append("eplb")
monkeypatch.setattr(
mrv2,
"pp_receive",
lambda *args, **kwargs: (
torch.zeros((2, 1), dtype=torch.long),
torch.ones(2, dtype=torch.int32),
torch.zeros(2, dtype=torch.int32),
),
runner.req_states = SimpleNamespace()
def fake_receive(*args, **kwargs):
events.append("receive")
# all_decode_next=True, so model_state.postprocess_state is skipped.
return True
runner.pp_handler = SimpleNamespace(receive=fake_receive)
runner.postprocess_num_computed_tokens = lambda *args, **kwargs: events.append(
"postprocess_num_computed_tokens"
)
runner.eplb.step = lambda *args, **kwargs: events.append("eplb")
output = mrv2.GPUModelRunner.sample_tokens(runner, None)
assert output in (EMPTY_MODEL_RUNNER_OUTPUT, None)
assert events == ["postprocess", "eplb"]
assert events == ["receive", "postprocess_num_computed_tokens", "eplb"]