diff --git a/tests/v1/worker/test_gpu_batch_shard.py b/tests/v1/worker/test_gpu_batch_shard.py index 09972b9ff6..1309686d53 100644 --- a/tests/v1/worker/test_gpu_batch_shard.py +++ b/tests/v1/worker/test_gpu_batch_shard.py @@ -96,7 +96,6 @@ def _make_batch( num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32), is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_), has_prefill=False, - max_seq_len_np=None, input_ids=torch.zeros(num_tokens, dtype=torch.int32, device=DEVICE), positions=torch.arange(num_tokens, dtype=torch.int64, device=DEVICE), is_padding=torch.zeros(num_tokens, dtype=torch.bool, device=DEVICE), diff --git a/tests/v1/worker/test_pp_utils.py b/tests/v1/worker/test_pp_utils.py new file mode 100644 index 0000000000..cfc92479f5 --- /dev/null +++ b/tests/v1/worker/test_pp_utils.py @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Which rows the PP sampled-token broadcast must carry.""" + +from unittest.mock import Mock + +import numpy as np + +from vllm.v1.worker.gpu import pp_utils + + +def _batch(num_computed, prefill_len, num_scheduled): + return Mock( + num_reqs=len(num_computed), + num_computed_tokens_np=np.array(num_computed, dtype=np.int32), + prefill_len_np=np.array(prefill_len, dtype=np.int32), + num_scheduled_tokens=np.array(num_scheduled, dtype=np.int32), + ) + + +def test_excludes_non_final_prefill_chunks(): + """Unchanged behaviour: a chunk that does not finish its prefill is skipped.""" + # Row 0 is a middle prefill chunk and produces no sample; row 1 finishes its + # prefill this step and therefore does. + batch = _batch( + num_computed=[512, 1000], + prefill_len=[4096, 1004], + num_scheduled=[448, 4], + ) + + mask = pp_utils.compute_need_sampled_mask(batch) + + assert mask is not None + assert mask.tolist() == [False, True] + + +def test_none_when_no_row_samples(): + """Unchanged behaviour: an all-prefill batch needs no broadcast at all.""" + batch = _batch( + num_computed=[0, 512], + prefill_len=[4096, 4096], + num_scheduled=[448, 448], + ) + + assert pp_utils.compute_need_sampled_mask(batch) is None + + +def test_keeps_decoding_request_past_its_length_cap(): + """A decoding request must never be dropped from the broadcast. + + Speculative decoding advances `num_computed_tokens` several tokens per step, + so it can overrun `prompt_len + max_tokens` while the scheduler is still + running the request. Predicting "this one is finishing" and skipping its + broadcast freezes the earlier pipeline stages' `last_sampled_tokens` and + `draft_tokens` while the last rank keeps advancing its own, and the stages + then diverge permanently. + """ + batch = _batch( + # 14176 computed tokens is already past this request's own + # prompt_len + max_tokens; the scheduler is still running it. + num_computed=[14176], + prefill_len=[12175], + num_scheduled=[8], + ) + + mask = pp_utils.compute_need_sampled_mask(batch) + + assert mask is not None + assert mask.tolist() == [True] + + +def test_decode_row_ahead_of_a_prefill_chunk(): + """Row order does not matter: only whether the row finishes its prefill.""" + batch = _batch( + num_computed=[10, 512], + prefill_len=[8, 4096], + num_scheduled=[1, 448], + ) + + mask = pp_utils.compute_need_sampled_mask(batch) + + assert mask is not None + assert mask.tolist() == [True, False] diff --git a/vllm/v1/worker/gpu/input_batch.py b/vllm/v1/worker/gpu/input_batch.py index 9a574a4faa..2a38c3d710 100644 --- a/vllm/v1/worker/gpu/input_batch.py +++ b/vllm/v1/worker/gpu/input_batch.py @@ -85,9 +85,6 @@ class InputBatch: # == np.any(is_prefilling_np) has_prefill: bool - # [num_reqs] only populated when pipeline parallelism is enabled. - max_seq_len_np: np.ndarray | None - # [num_tokens_after_padding] input_ids: torch.Tensor # [num_tokens_after_padding] @@ -194,7 +191,6 @@ class InputBatch: num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32), is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_), has_prefill=False, - max_seq_len_np=None, input_ids=input_ids, positions=positions, is_padding=is_padding, diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 0ee05328c9..89ac0aebf8 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1311,11 +1311,6 @@ class GPUModelRunner(LoRAModelRunnerMixin): ) seq_lens_cpu_upper_bound = torch.from_numpy(seq_lens_cpu_upper_bound_np) - max_seq_len_np = None - if self.use_pp: - # max_seq_len is only consumed by the PP `compute_need_sampled_mask` - max_seq_len_np = self.req_states.max_seq_len[idx_mapping_np] - prompt_lens = None if self.model_config.rswa_window is not None: # prompt_lens is only used in R-SWA case. @@ -1344,7 +1339,6 @@ class GPUModelRunner(LoRAModelRunnerMixin): num_computed_prefill_tokens_np=batch_req_state.num_computed_prefill_tokens_np, is_prefilling_np=batch_req_state.is_prefilling_np, has_prefill=batch_req_state.has_prefill, - max_seq_len_np=max_seq_len_np, input_ids=self.input_buffers.input_ids[:num_tokens_after_padding], positions=self.input_buffers.positions[:num_tokens_after_padding], is_padding=self.input_buffers.is_padding[:num_tokens_after_padding], diff --git a/vllm/v1/worker/gpu/pcp_manager.py b/vllm/v1/worker/gpu/pcp_manager.py index f8c5a70550..146320be13 100644 --- a/vllm/v1/worker/gpu/pcp_manager.py +++ b/vllm/v1/worker/gpu/pcp_manager.py @@ -575,9 +575,6 @@ class PCPManager: num_computed_prefill_tokens_np=local_num_computed_prefill_tokens_np, is_prefilling_np=local_is_prefilling_np, has_prefill=bool(local_is_prefilling_np.any()), - max_seq_len_np=global_batch.max_seq_len_np[local_to_global_batch_req_idx_np] - if global_batch.max_seq_len_np is not None - else None, input_ids=input_buffers.input_ids[:num_local_tokens_padded], positions=input_buffers.positions[:num_local_tokens_padded], is_padding=is_padding, diff --git a/vllm/v1/worker/gpu/pp_utils.py b/vllm/v1/worker/gpu/pp_utils.py index 9b1786cbe4..cf52a6d382 100644 --- a/vllm/v1/worker/gpu/pp_utils.py +++ b/vllm/v1/worker/gpu/pp_utils.py @@ -34,19 +34,15 @@ class PendingRecv: def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None: """Return a bool array of shape `[input_batch.num_reqs]` marking requests - with outputs that might be needed in a subsequent (decode) step. - Returns None if no sampled outputs are needed in the requests' next step.""" + that produce a sampled token this step, and therefore must have that token + (and the draft block proposed from it) propagated to the earlier PP stages. + Returns None if no request in the batch produces a sample.""" old_computed = input_batch.num_computed_tokens_np prefill_len = input_batch.prefill_len_np - max_seq_len = input_batch.max_seq_len_np - assert max_seq_len is not None # always populated under PP # Exclude non-final prefill chunks (they don't produce a sample). produces_sample = old_computed + input_batch.num_scheduled_tokens >= prefill_len - # Exclude requests that we know are finished. - not_finishing = np.maximum(old_computed, prefill_len) + 1 < max_seq_len - need_sampled_mask = produces_sample & not_finishing - return need_sampled_mask if need_sampled_mask.any() else None + return produces_sample if produces_sample.any() else None class PPHandler: