[Bugfix][MRV2] Fix speculative warmup deadlock with pipeline parallelism (#62129)
Signed-off-by: chenjr97 <chenjr97@mail2.sysu.edu.cn> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: vllm-agent <claw@inferact.ai>
This commit is contained in:
co-authored by
Nick Hill
vllm-agent
parent
da9134bc89
commit
7c9042c05a
@@ -88,6 +88,7 @@ def _make_runner(
|
|||||||
"""Stub model runner exposing only what the warmup entry points read."""
|
"""Stub model runner exposing only what the warmup entry points read."""
|
||||||
return SimpleNamespace(
|
return SimpleNamespace(
|
||||||
num_speculative_steps=num_spec_steps,
|
num_speculative_steps=num_spec_steps,
|
||||||
|
speculative_config=SimpleNamespace() if num_spec_steps else None,
|
||||||
adaptive_verification=None,
|
adaptive_verification=None,
|
||||||
rejection_sampler=None,
|
rejection_sampler=None,
|
||||||
decode_query_len=num_spec_steps + 1,
|
decode_query_len=num_spec_steps + 1,
|
||||||
|
|||||||
@@ -74,6 +74,7 @@ def test_kernel_warmup_restores_uncalibrated_adaptive_manager(monkeypatch, fail_
|
|||||||
runner = SimpleNamespace(
|
runner = SimpleNamespace(
|
||||||
adaptive_verification=manager,
|
adaptive_verification=manager,
|
||||||
rejection_sampler=rejection_sampler,
|
rejection_sampler=rejection_sampler,
|
||||||
|
speculative_config=SimpleNamespace(),
|
||||||
)
|
)
|
||||||
|
|
||||||
def run_steps(model_runner, execute, sample, greedy=False):
|
def run_steps(model_runner, execute, sample, greedy=False):
|
||||||
@@ -91,3 +92,27 @@ def test_kernel_warmup_restores_uncalibrated_adaptive_manager(monkeypatch, fail_
|
|||||||
assert runner.adaptive_verification is manager
|
assert runner.adaptive_verification is manager
|
||||||
assert manager.cost_tables is None
|
assert manager.cost_tables is None
|
||||||
assert rejection_sampler.enable_adaptive_verification
|
assert rejection_sampler.enable_adaptive_verification
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("speculative", [False, True])
|
||||||
|
def test_kernel_warmup_runs_same_rounds_on_every_pp_stage(monkeypatch, speculative):
|
||||||
|
"""PP receivers need a matching sender for every speculative warmup round."""
|
||||||
|
rounds = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
warmup,
|
||||||
|
"_warmup_kernels",
|
||||||
|
lambda runner, execute, sample, greedy=False: rounds.append(greedy),
|
||||||
|
)
|
||||||
|
for is_last_pp_rank in (False, True):
|
||||||
|
rounds.clear()
|
||||||
|
runner = SimpleNamespace(
|
||||||
|
adaptive_verification=None,
|
||||||
|
speculative_config=SimpleNamespace() if speculative else None,
|
||||||
|
rejection_sampler=(
|
||||||
|
SimpleNamespace(enable_adaptive_verification=False)
|
||||||
|
if is_last_pp_rank and speculative
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
)
|
||||||
|
warmup.warmup_kernels(runner, _fail, _fail)
|
||||||
|
assert rounds == ([False, True] if speculative else [False])
|
||||||
|
|||||||
@@ -238,7 +238,8 @@ def warmup_kernels(
|
|||||||
rejection_sampler.enable_adaptive_verification = False
|
rejection_sampler.enable_adaptive_verification = False
|
||||||
try:
|
try:
|
||||||
_warmup_kernels(model_runner, worker_execute_model, worker_sample_tokens)
|
_warmup_kernels(model_runner, worker_execute_model, worker_sample_tokens)
|
||||||
if rejection_sampler is not None:
|
# All PP ranks must run the same forwards to match sends and receives.
|
||||||
|
if model_runner.speculative_config is not None:
|
||||||
# Greedy batches skip logits processing, so rejection kernels
|
# Greedy batches skip logits processing, so rejection kernels
|
||||||
# also see model-dtype logits instead of FP32.
|
# also see model-dtype logits instead of FP32.
|
||||||
_warmup_kernels(
|
_warmup_kernels(
|
||||||
|
|||||||
Reference in New Issue
Block a user