From e15f20258b4448999eee5c133577bf86bf1693a2 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Tue, 2 Jun 2026 14:02:01 -0700 Subject: [PATCH] [ModelRunnerV2] Avoid pipeline parallel bubbles (#42187) Signed-off-by: Nick Hill --- .buildkite/test_areas/model_runner_v2.yaml | 2 + tests/v1/distributed/test_pp_dp_v2.py | 178 +++++++++++++++ .../worker/test_gpu_model_runner_v2_eplb.py | 27 ++- vllm/config/vllm.py | 10 +- vllm/distributed/parallel_state.py | 19 ++ vllm/v1/core/sched/async_scheduler.py | 6 + vllm/v1/core/sched/scheduler.py | 10 + vllm/v1/engine/core.py | 8 +- vllm/v1/request.py | 4 + vllm/v1/worker/gpu/buffer_utils.py | 22 +- vllm/v1/worker/gpu/input_batch.py | 44 +++- vllm/v1/worker/gpu/model_runner.py | 135 ++++++++---- vllm/v1/worker/gpu/model_states/default.py | 5 +- vllm/v1/worker/gpu/model_states/interface.py | 9 +- .../worker/gpu/model_states/mamba_hybrid.py | 43 +++- vllm/v1/worker/gpu/model_states/whisper.py | 5 +- vllm/v1/worker/gpu/pp_utils.py | 205 +++++++++++++++--- vllm/v1/worker/gpu/sample/prompt_logprob.py | 8 +- vllm/v1/worker/gpu/states.py | 19 +- 19 files changed, 615 insertions(+), 144 deletions(-) create mode 100644 tests/v1/distributed/test_pp_dp_v2.py diff --git a/.buildkite/test_areas/model_runner_v2.yaml b/.buildkite/test_areas/model_runner_v2.yaml index 2964762b34..617c80b2fe 100644 --- a/.buildkite/test_areas/model_runner_v2.yaml +++ b/.buildkite/test_areas/model_runner_v2.yaml @@ -94,11 +94,13 @@ steps: - vllm/v1/worker/gpu_worker.py - tests/distributed/test_pipeline_parallel.py - tests/distributed/test_pp_cudagraph.py + - tests/v1/distributed/test_pp_dp_v2.py commands: - set -x - export VLLM_USE_V2_MODEL_RUNNER=1 - pytest -v -s distributed/test_pipeline_parallel.py -k "not ray and not Jamba" - pytest -v -s distributed/test_pp_cudagraph.py -k "not ray" + - pytest -v -s v1/distributed/test_pp_dp_v2.py - label: Model Runner V2 Spec Decode device: h200_35gb diff --git a/tests/v1/distributed/test_pp_dp_v2.py b/tests/v1/distributed/test_pp_dp_v2.py new file mode 100644 index 0000000000..3533154997 --- /dev/null +++ b/tests/v1/distributed/test_pp_dp_v2.py @@ -0,0 +1,178 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""V2 ModelRunner + pipeline parallel + data parallel integration tests. + +Covers the interaction between the V2 model runner's PP sampled-token +broadcast and the DP per-step all-reduce across a few concurrency +regimes. Requires 4 GPUs (DP=2, PP=2, TP=1) on CUDA. +""" + +import asyncio +import contextlib +import os +from contextlib import ExitStack + +import pytest + +from vllm import SamplingParams +from vllm.engine.arg_utils import AsyncEngineArgs +from vllm.platforms import current_platform +from vllm.sampling_params import RequestOutputKind +from vllm.v1.engine.async_llm import AsyncLLM + +PP_DP_MODEL = "ibm-research/PowerMoE-3b" # smallest cached MoE that supports PP +PROMPT = "This is a test of data parallel and pipeline parallel together" + + +def _gpu_skip_reason() -> str | None: + if not current_platform.is_cuda(): + return "requires CUDA" + n = current_platform.device_count() + if n < 4: + return f"requires 4 GPUs, got {n}" + return None + + +_GPU_SKIP = _gpu_skip_reason() + +pytestmark = [ + pytest.mark.skipif( + os.environ.get("VLLM_USE_V2_MODEL_RUNNER", "0") != "1", + reason="VLLM_USE_V2_MODEL_RUNNER=1 required", + ), + pytest.mark.skipif(_GPU_SKIP is not None, reason=_GPU_SKIP or ""), +] + + +def _engine_args(async_scheduling: bool) -> AsyncEngineArgs: + return AsyncEngineArgs( + model=PP_DP_MODEL, + pipeline_parallel_size=2, + data_parallel_size=2, + data_parallel_backend="mp", + tensor_parallel_size=1, + max_model_len=4096, + max_num_batched_tokens=2048, + max_num_seqs=256, + async_scheduling=async_scheduling, + enable_prefix_caching=False, + enforce_eager=False, + enable_expert_parallel=False, + ) + + +async def _generate(engine: AsyncLLM, prompt: str, max_tokens: int) -> int: + """Run one streaming completion and return the number of tokens it yielded.""" + sampling_params = SamplingParams( + max_tokens=max_tokens, + ignore_eos=True, + output_kind=RequestOutputKind.DELTA, + temperature=0.0, + ) + request_id = f"req-{id(prompt):x}-{max_tokens}" + total = 0 + async for out in engine.generate( + request_id=request_id, prompt=prompt, sampling_params=sampling_params + ): + total += len(out.outputs[0].token_ids) + return total + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_scheduling", [True, False]) +async def test_pp_dp_v2_low_concurrency(async_scheduling: bool): + """A single in-flight request at a time, repeated, to exercise the + PP slot ring under empty batches between decodes.""" + with ExitStack() as after: + engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling)) + after.callback(engine.shutdown) + + for _ in range(4): + n = await _generate(engine, PROMPT, max_tokens=16) + assert n == 16 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_scheduling", [True, False]) +async def test_pp_dp_v2_mid_concurrency(async_scheduling: bool): + """64 concurrent requests, staggered, to exercise the steady-state + DP all-reduce + PP slot-ring path.""" + with ExitStack() as after: + engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling)) + after.callback(engine.shutdown) + + async def _one(i: int) -> int: + await asyncio.sleep(0.01 * i) # stagger so DP load-balances + return await _generate(engine, f"{PROMPT} {i}", max_tokens=64) + + results = await asyncio.gather(*[_one(i) for i in range(64)]) + assert all(n == 64 for n in results), results + + +@pytest.mark.asyncio +async def test_pp_dp_v2_abort_mid_decode(): + """Cancel half the in-flight requests mid-stream and confirm the + engine survives the abort storm.""" + + with ExitStack() as after: + engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling=True)) + after.callback(engine.shutdown) + + async def _maybe_cancel(i: int): + sampling_params = SamplingParams( + max_tokens=64, + ignore_eos=True, + output_kind=RequestOutputKind.DELTA, + temperature=0.0, + ) + request_id = f"abort-req-{i}" + count = 0 + cancel_at = 4 if i % 2 == 0 else 64 + async for out in engine.generate( + request_id=request_id, + prompt=f"{PROMPT} {i}", + sampling_params=sampling_params, + ): + count += len(out.outputs[0].token_ids) + if count >= cancel_at: + break + return count, i + + results = await asyncio.gather(*[_maybe_cancel(i) for i in range(32)]) + for count, i in results: + if i % 2 == 0: + assert count >= 4 + else: + assert count == 64 + + # Engine must still serve after the abort storm. + final = await _generate(engine, "post-abort warmup", max_tokens=8) + assert final == 8 + + +@pytest.mark.asyncio +async def test_pp_dp_v2_pause_resume(): + """Pause an engine with a request in flight, then resume and confirm + new requests still work.""" + + with ExitStack() as after: + engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling=True)) + after.callback(engine.shutdown) + + # Start a long-running generation, let some decoding happen, then + # pause (abort mode) and confirm the in-flight task terminates. + inflight = asyncio.create_task(_generate(engine, PROMPT, max_tokens=128)) + await asyncio.sleep(0.5) + + assert not await engine.is_paused() + await engine.pause_generation(mode="abort") + assert await engine.is_paused() + + with contextlib.suppress(Exception): + await inflight + + await engine.resume_generation() + assert not await engine.is_paused() + + n = await _generate(engine, PROMPT, max_tokens=8) + assert n == 8 diff --git a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py index c2a800bd99..1db07baf93 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py +++ b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py @@ -167,7 +167,9 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch): events = [] runner = _make_runner(is_last_pp_rank=False, num_speculative_steps=0) runner.execute_model_state = SimpleNamespace( - input_batch=SimpleNamespace(num_reqs=2), + input_batch=SimpleNamespace( + num_reqs=2, idx_mapping=torch.zeros(2, dtype=torch.int32) + ), attn_metadata=None, slot_mappings_by_layer=None, hidden_states=None, @@ -175,18 +177,19 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch): finished_req_ids=set(), num_tokens_across_dp=None, ) - runner.postprocess = lambda *args, **kwargs: events.append("postprocess") - runner.eplb.step = lambda *args, **kwargs: events.append("eplb") - monkeypatch.setattr( - mrv2, - "pp_receive", - lambda *args, **kwargs: ( - torch.zeros((2, 1), dtype=torch.long), - torch.ones(2, dtype=torch.int32), - torch.zeros(2, dtype=torch.int32), - ), + runner.req_states = SimpleNamespace() + + def fake_receive(*args, **kwargs): + events.append("receive") + # all_decode_next=True, so model_state.postprocess_state is skipped. + return True + + runner.pp_handler = SimpleNamespace(receive=fake_receive) + runner.postprocess_num_computed_tokens = lambda *args, **kwargs: events.append( + "postprocess_num_computed_tokens" ) + runner.eplb.step = lambda *args, **kwargs: events.append("eplb") output = mrv2.GPUModelRunner.sample_tokens(runner, None) assert output in (EMPTY_MODEL_RUNNER_OUTPUT, None) - assert events == ["postprocess", "eplb"] + assert events == ["receive", "postprocess_num_computed_tokens", "eplb"] diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index b772a25028..4d80078a01 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -498,9 +498,13 @@ class VllmConfig: # PP requires PP-size concurrent batches to fill the pipeline. # Async scheduling requires 2 concurrent batches to overlap. pp_size = self.parallel_config.pipeline_parallel_size - if pp_size > 1: - return pp_size - return 2 if self.scheduler_config.async_scheduling else 1 + if self.scheduler_config.async_scheduling: + if self.use_v2_model_runner: + return pp_size + 1 + # V1 Model Runner does not fully support async scheduling with PP. + if pp_size <= 1: + return 2 + return pp_size @property def num_speculative_tokens(self) -> int: diff --git a/vllm/distributed/parallel_state.py b/vllm/distributed/parallel_state.py index 712167c601..331e0684e3 100644 --- a/vllm/distributed/parallel_state.py +++ b/vllm/distributed/parallel_state.py @@ -359,6 +359,9 @@ class GroupCoordinator: assert self_cpu_group is not None assert self_device_group is not None + self.group_ranks = group_ranks + self.torch_distributed_backend = torch_distributed_backend + self.cpu_group = self_cpu_group self.device_group = self_device_group @@ -406,6 +409,22 @@ class GroupCoordinator: and getattr(self.device_communicator, "supports_tensor_dict", False) ) + def make_sibling_device_group(self, group_desc: str | None = None) -> ProcessGroup: + """Create a new device-side ProcessGroup with the same per-rank membership + as this coordinator's `device_group`, but backed by a distinct communicator. + This is a collective call: every world rank must invoke it. Used where we + want to issue ops that can run concurrently with ops on `device_group`. + """ + sibling: ProcessGroup | None = None + for ranks in self.group_ranks: + pg = torch.distributed.new_group( + ranks, backend=self.torch_distributed_backend, group_desc=group_desc + ) + if self.rank in ranks: + sibling = pg + assert sibling is not None + return sibling + def create_mq_broadcaster( self, writer_rank=0, external_writer_handle=None, blocking=True ): diff --git a/vllm/v1/core/sched/async_scheduler.py b/vllm/v1/core/sched/async_scheduler.py index cb61bcabd3..2fd22f4c0c 100644 --- a/vllm/v1/core/sched/async_scheduler.py +++ b/vllm/v1/core/sched/async_scheduler.py @@ -14,6 +14,7 @@ class AsyncScheduler(Scheduler): super().__init__(*args, **kwargs) # reusable read-only placeholder list for speculative decoding. self._spec_token_placeholders: list[int] = [-1] * self.num_spec_tokens + self.pp_size = self.parallel_config.pipeline_parallel_size def _update_after_schedule(self, scheduler_output: SchedulerOutput) -> None: super()._update_after_schedule(scheduler_output) @@ -34,6 +35,11 @@ class AsyncScheduler(Scheduler): # We will update the actual spec token ids in the worker process. request.spec_token_ids = self._spec_token_placeholders + if self.use_v2_model_runner: + # Set the next step index in which this request is eligible to be + # scheduled for decode (for PP microbatching). + request.next_decode_eligible_step = self.current_step + self.pp_size + def _update_request_with_output( self, request: Request, new_token_ids: list[int] ) -> tuple[list[int], bool]: diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index ab932c8a6d..f76ad05607 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -248,6 +248,9 @@ class Scheduler(SchedulerInterface): self.use_pp = self.parallel_config.pipeline_parallel_size > 1 self.use_v2_model_runner = vllm_config.use_v2_model_runner + # Scheduler iteration counter. Drives the V2+PP+async decode-throttle + # cadence (`next_decode_eligible_step`). + self.current_step = 0 self.scheduler_reserve_full_isl = ( self.scheduler_config.scheduler_reserve_full_isl ) @@ -333,6 +336,7 @@ class Scheduler(SchedulerInterface): return num_new_tokens def schedule(self) -> SchedulerOutput: + self.current_step += 1 # NOTE(woosuk) on the scheduling algorithm: # There's no "decoding phase" nor "prefill phase" in the scheduler. # Each request just has the num_computed_tokens and @@ -388,6 +392,12 @@ class Scheduler(SchedulerInterface): req_index += 1 continue + if self.current_step < request.next_decode_eligible_step: + # V2+PP+async: enforce `pp_size` steps between same-req decodes + # to match worker-side sampled-tokens broadcast slot ring cadence. + req_index += 1 + continue + num_new_tokens = ( request.num_tokens_with_spec + request.num_output_placeholders diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 60583ea588..50fd98e1fd 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -534,14 +534,12 @@ class EngineCore: if not deferred_scheduler_output: # Add this step's future to the queue. batch_queue.appendleft((future, scheduler_output, exec_future)) - if ( - model_executed - and len(batch_queue) < self.batch_queue_size - and not batch_queue[-1][0].done() + if len(batch_queue) < self.batch_queue_size and ( + model_executed or self.scheduler.has_requests() ): # Don't block on next worker response unless the queue is full # or there are no more requests to schedule. - return None, True + return None, model_executed elif not batch_queue: # Queue is empty. We should not reach here since this method should diff --git a/vllm/v1/request.py b/vllm/v1/request.py index 26cc82fc4a..44246e70a8 100644 --- a/vllm/v1/request.py +++ b/vllm/v1/request.py @@ -141,6 +141,10 @@ class Request: self.num_output_placeholders = 0 self.async_tokens_to_discard = 0 + # V2+PP+async: Enforces `pp_size` cadence between same-request decode steps + # so the worker's broadcast slot ring stays consistent. + self.next_decode_eligible_step = 0 + self.spec_token_ids: list[int] = [] self.num_computed_tokens = 0 self.cache_salt: str | None = cache_salt diff --git a/vllm/v1/worker/gpu/buffer_utils.py b/vllm/v1/worker/gpu/buffer_utils.py index 5963790a77..e4497de43a 100644 --- a/vllm/v1/worker/gpu/buffer_utils.py +++ b/vllm/v1/worker/gpu/buffer_utils.py @@ -13,6 +13,15 @@ from vllm.utils.torch_utils import ( get_accelerator_view_from_cpu_tensor, ) +# Default round-robin depth for the UVA buffer pools. Must be >= the number of +# concurrent in-flight steps (engine batch_queue_size). +_DEFAULT_MAX_CONCURRENCY = 2 + + +def set_default_max_concurrency(n: int) -> None: + global _DEFAULT_MAX_CONCURRENCY + _DEFAULT_MAX_CONCURRENCY = max(2, n) + def async_copy_to_gpu( x: torch.Tensor | np.ndarray, @@ -47,8 +56,10 @@ class UvaBufferPool: self, size: int | Sequence[int], dtype: torch.dtype, - max_concurrency: int = 2, + max_concurrency: int | None = None, ): + if max_concurrency is None: + max_concurrency = _DEFAULT_MAX_CONCURRENCY self.size = size self.dtype = dtype self.max_concurrency = max_concurrency @@ -80,7 +91,10 @@ class UvaBufferPool: class UvaBackedTensor: def __init__( - self, size: int | Sequence[int], dtype: torch.dtype, max_concurrency: int = 2 + self, + size: int | Sequence[int], + dtype: torch.dtype, + max_concurrency: int | None = None, ): self.dtype = dtype @@ -104,9 +118,11 @@ class StagedWriteTensor: size: int | Sequence[int], dtype: torch.dtype, device: torch.device, - max_concurrency: int = 2, + max_concurrency: int | None = None, uva_instead_of_gpu: bool = False, ): + if max_concurrency is None: + max_concurrency = _DEFAULT_MAX_CONCURRENCY supported_dtypes = [torch.int32, torch.int64, torch.float32] if dtype not in supported_dtypes: raise ValueError( diff --git a/vllm/v1/worker/gpu/input_batch.py b/vllm/v1/worker/gpu/input_batch.py index b253d7d8c0..f905d09e45 100644 --- a/vllm/v1/worker/gpu/input_batch.py +++ b/vllm/v1/worker/gpu/input_batch.py @@ -67,9 +67,18 @@ class InputBatch: seq_lens_cpu_upper_bound: torch.Tensor # [num_reqs] dcp_local_seq_lens: torch.Tensor | None - # [num_reqs] CPU bool array. + # [num_reqs] + num_computed_tokens_np: np.ndarray + # [num_reqs] + prefill_len_np: np.ndarray + # [num_reqs] + num_computed_prefill_tokens_np: np.ndarray + # [num_reqs] CPU bool array == (num_computed_prefill_tokens_np < prefill_len_np). is_prefilling_np: np.ndarray + # [num_reqs] only populated when pipeline parallelism is enabled. + max_seq_len_np: np.ndarray | None + # [num_tokens_after_padding] input_ids: torch.Tensor # [num_tokens_after_padding] @@ -148,7 +157,11 @@ class InputBatch: seq_lens=seq_lens, seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, dcp_local_seq_lens=None, + num_computed_tokens_np=np.zeros(num_reqs, dtype=np.int32), + prefill_len_np=np.zeros(num_reqs, dtype=np.int32), + num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32), is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_), + max_seq_len_np=None, input_ids=input_ids, positions=positions, logits_indices=logits_indices, @@ -438,6 +451,9 @@ def _post_update_kernel( ): req_id = tl.program_id(0) req_state_idx = tl.load(idx_mapping_ptr + req_id) + if req_state_idx < 0: + # Filter rows with negative index entries. + return total_len = tl.load(total_len_ptr + req_state_idx) num_sampled = tl.load(num_sampled_ptr + req_id) @@ -464,18 +480,22 @@ def _post_update_kernel( count = tl.load(token_ptr) tl.store(token_ptr, count + 1) - query_start = tl.load(query_start_loc_ptr + req_id) - query_end = tl.load(query_start_loc_ptr + req_id + 1) - query_len = query_end - query_start + if query_start_loc_ptr is None: + query_len = 0 + else: + query_start = tl.load(query_start_loc_ptr + req_id) + query_end = tl.load(query_start_loc_ptr + req_id + 1) + query_len = query_end - query_start num_rejected = tl.load(num_rejected_ptr + req_id) - num_computed = tl.load(num_computed_tokens_ptr + req_state_idx) - num_computed += query_len - num_rejected - tl.store(num_computed_tokens_ptr + req_state_idx, num_computed) + computed_delta = query_len - num_rejected + if computed_delta != 0: + num_computed = tl.load(num_computed_tokens_ptr + req_state_idx) + tl.store(num_computed_tokens_ptr + req_state_idx, num_computed + computed_delta) def post_update( - # [num_reqs] + # [num_reqs] batch_idx -> req_state_idx; negative index means skip. idx_mapping: torch.Tensor, # [max_num_reqs] num_computed_tokens: torch.Tensor, @@ -490,7 +510,7 @@ def post_update( # [num_reqs] num_rejected: torch.Tensor, # [num_reqs + 1] - query_start_loc: torch.Tensor, + query_start_loc: torch.Tensor | None, # [max_num_reqs, max_model_len] all_token_ids: torch.Tensor, # [max_num_reqs] @@ -516,7 +536,7 @@ def post_update( @triton.jit -def _post_update_pool_kernel( +def _post_update_num_computed_tokens_kernel( idx_mapping_ptr, num_computed_tokens_ptr, query_start_loc_ptr, @@ -531,7 +551,7 @@ def _post_update_pool_kernel( tl.store(num_computed_tokens_ptr + req_state_idx, num_computed + query_len) -def post_update_pool( +def post_update_num_computed_tokens( # [num_reqs] idx_mapping: torch.Tensor, # [max_num_reqs] @@ -540,7 +560,7 @@ def post_update_pool( query_start_loc: torch.Tensor, ) -> None: num_reqs = idx_mapping.shape[0] - _post_update_pool_kernel[(num_reqs,)]( + _post_update_num_computed_tokens_kernel[(num_reqs,)]( idx_mapping, num_computed_tokens, query_start_loc, diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 532b3c6c0b..2e3133822f 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -61,7 +61,10 @@ from vllm.v1.worker.gpu.attn_utils import ( init_kv_cache, ) from vllm.v1.worker.gpu.block_table import BlockTables -from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu +from vllm.v1.worker.gpu.buffer_utils import ( + async_copy_to_gpu, + set_default_max_concurrency, +) from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens from vllm.v1.worker.gpu.cudagraph_utils import ( BatchExecutionDescriptor, @@ -77,7 +80,7 @@ from vllm.v1.worker.gpu.input_batch import ( expand_idx_mapping, get_num_sampled_and_rejected, post_update, - post_update_pool, + post_update_num_computed_tokens, prepare_pos_seq_lens, prepare_prefill_inputs, ) @@ -90,7 +93,7 @@ from vllm.v1.worker.gpu.lora_utils import LoraState from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache from vllm.v1.worker.gpu.model_states import init_model_state from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner -from vllm.v1.worker.gpu.pp_utils import pp_broadcast, pp_receive +from vllm.v1.worker.gpu.pp_utils import PPHandler from vllm.v1.worker.gpu.sample.output import SamplerOutput from vllm.v1.worker.gpu.sample.prompt_logprob import PromptLogprobsWorker from vllm.v1.worker.gpu.sample.sampler import Sampler @@ -149,6 +152,13 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.is_first_pp_rank = get_pp_group().is_first_rank self.is_last_pp_rank = get_pp_group().is_last_rank + # Size the UVA buffer pools to the max number of concurrent in-flight + # steps. Must run before any pooled buffer is constructed + set_default_max_concurrency(vllm_config.max_concurrent_batches) + + # PP broadcast/recv helper. Runs the collective on a side stream. + self.pp_handler: PPHandler | None = None + # Persistent buffer for intermediate tensors (non-first PP ranks). self.intermediate_tensors: IntermediateTensors | None = None @@ -210,6 +220,13 @@ class GPUModelRunner(LoRAModelRunnerMixin): device=self.device, ) + if self.use_pp: + self.pp_handler = PPHandler( + max_num_reqs=self.max_num_reqs, + num_speculative_steps=self.num_speculative_steps, + device=self.device, + ) + self.sampler: Sampler | None = None self.rejection_sampler: RejectionSampler | None = None self.prompt_logprobs_worker: PromptLogprobsWorker | None = None @@ -688,8 +705,11 @@ class GPUModelRunner(LoRAModelRunnerMixin): return cuda_graph_size def _remove_request(self, req_id: str) -> bool: - if not self.req_states.remove_request(req_id): + req_idx = self.req_states.remove_request(req_id) + if req_idx is None: return False + if self.pp_handler is not None: + self.pp_handler.on_req_idx_freed(req_idx) if self.encoder_cache is not None: self.encoder_cache.remove_request(req_id) if self.prompt_logprobs_worker is not None: @@ -710,6 +730,14 @@ class GPUModelRunner(LoRAModelRunnerMixin): for mm_hash in scheduler_output.free_encoder_mm_hashes: self.encoder_cache.free_encoder_cache(mm_hash) + def update_pp_decode_requests(self): + # For non-last PP ranks, update decode requests with sampler output from + # the prior step in which they were scheduled (pp_size steps ago). + if self.pp_handler is not None: + outputs = self.pp_handler.get_prev_sampled_outputs() + if outputs is not None: + self.postprocess_sampled(**outputs) + def add_requests(self, scheduler_output: SchedulerOutput) -> None: for new_req_data in scheduler_output.scheduled_new_reqs: assert new_req_data.prompt_token_ids is not None @@ -722,11 +750,13 @@ class GPUModelRunner(LoRAModelRunnerMixin): self._remove_request(req_id) prompt_len = len(new_req_data.prompt_token_ids) + sampling_params = new_req_data.sampling_params self.req_states.add_request( req_id=req_id, prompt_len=prompt_len, all_token_ids=new_req_data.prefill_token_ids, num_computed_tokens=new_req_data.num_computed_tokens, + max_tokens=sampling_params.max_tokens if sampling_params else 1, # type: ignore[arg-type] ) req_index = self.req_states.req_id_to_index[req_id] @@ -769,7 +799,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): req_index, req_new_block_ids, overwrite=False ) - # Update num_computed_prefill_tokens. + # Update CPU num_computed_prefill_tokens. np.minimum( self.req_states.num_computed_tokens_np, self.req_states.prefill_len.np, @@ -848,7 +878,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc) query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1] query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1] - is_prefilling_np = self.req_states.is_prefilling(idx_mapping_np) + prefill_len_np = self.req_states.prefill_len.np[idx_mapping_np] + computed_prefill_tokens_np = self.req_states.num_computed_prefill_tokens + num_computed_prefill_tokens_np = computed_prefill_tokens_np[idx_mapping_np] + is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np # Get prefill tokens if any. if np.any(is_prefilling_np): @@ -900,13 +933,19 @@ class GPUModelRunner(LoRAModelRunnerMixin): ) # CPU upper bound on seq_lens; padded entries left at zero. + num_computed_tokens_np = self.req_states.num_computed_tokens_np[idx_mapping_np] seq_lens_cpu_upper_bound_np = np.zeros(num_reqs_padded, dtype=np.int32) np.add( - self.req_states.num_computed_tokens_np[idx_mapping_np], + num_computed_tokens_np, num_scheduled_tokens, out=seq_lens_cpu_upper_bound_np[:num_reqs], ) seq_lens_cpu_upper_bound = torch.from_numpy(seq_lens_cpu_upper_bound_np) + + max_seq_len_np = None + if self.use_pp: + # max_seq_len is only consumed by the PP `compute_need_sampled_mask` + max_seq_len_np = self.req_states.max_seq_len[idx_mapping_np] return InputBatch( req_ids=req_ids, num_reqs=num_reqs, @@ -925,7 +964,11 @@ class GPUModelRunner(LoRAModelRunnerMixin): seq_lens=seq_lens, seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, dcp_local_seq_lens=dcp_local_seq_lens, + num_computed_tokens_np=num_computed_tokens_np, + prefill_len_np=prefill_len_np, + num_computed_prefill_tokens_np=num_computed_prefill_tokens_np, is_prefilling_np=is_prefilling_np, + max_seq_len_np=max_seq_len_np, input_ids=self.input_buffers.input_ids[:num_tokens_after_padding], positions=self.input_buffers.positions[:num_tokens_after_padding], logits_indices=logits_indices, @@ -1005,12 +1048,13 @@ class GPUModelRunner(LoRAModelRunnerMixin): ) return sampler_output, num_sampled, num_rejected - def postprocess( + def postprocess_sampled( self, - input_batch: InputBatch, + idx_mapping: torch.Tensor, # May include -1 for masked entries sampled_tokens: torch.Tensor, num_sampled: torch.Tensor, num_rejected: torch.Tensor, + query_start_loc: torch.Tensor | None = None, ) -> None: # Update the number of computed tokens. if self.is_last_pp_rank: @@ -1019,19 +1063,19 @@ class GPUModelRunner(LoRAModelRunnerMixin): else: output_bin_counts = None post_update( - input_batch.idx_mapping, + idx_mapping, self.req_states.num_computed_tokens.gpu, self.req_states.last_sampled_tokens, output_bin_counts, sampled_tokens, num_sampled, num_rejected, - input_batch.query_start_loc, + query_start_loc, self.req_states.all_token_ids.gpu, self.req_states.total_len.gpu, ) - self.model_state.postprocess_state(input_batch, num_sampled) + self.model_state.postprocess_state(idx_mapping, num_sampled) @torch.inference_mode() def execute_model( @@ -1044,6 +1088,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): ) -> ModelRunnerOutput | IntermediateTensors | None: if not dummy_run: # Update the request states. + self.update_pp_decode_requests() self.finish_requests(scheduler_output) self.free_states(scheduler_output) self.add_requests(scheduler_output) @@ -1162,9 +1207,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): # NOTE(woosuk): We must call get_mm_embeddings even during dummy runs # to obtain inputs_embeds, because the compiled model expects this input. inputs_embeds = self.model_state.get_mm_embeddings( - scheduler_output.scheduled_encoder_inputs, - input_batch, - self.req_states, + scheduler_output.scheduled_encoder_inputs, input_batch ) model_inputs = { @@ -1184,12 +1227,13 @@ class GPUModelRunner(LoRAModelRunnerMixin): assert intermediate_tensors is not None assert self.intermediate_tensors is not None n = input_batch.num_tokens_after_padding - model_inputs["intermediate_tensors"] = IntermediateTensors( - { - k: v[:n].copy_(intermediate_tensors.tensors[k][:n]) - for k, v in self.intermediate_tensors.tensors.items() - } - ) + new_tensors = { + k: v[:n] + if dummy_run + else v[:n].copy_(intermediate_tensors.tensors[k][:n]) + for k, v in self.intermediate_tensors.tensors.items() + } + model_inputs["intermediate_tensors"] = IntermediateTensors(new_tensors) del intermediate_tensors # Run model. @@ -1281,10 +1325,15 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Non-last PP rank: hidden_states is None because this rank produced # IntermediateTensors instead of final hidden states. Receive the # sampled tokens broadcast from the last rank and update local state. - sampled, num_sampled, num_rejected = pp_receive( - input_batch.num_reqs, max_sample_len=self.num_speculative_steps + 1 - ) - self.postprocess(input_batch, sampled, num_sampled, num_rejected) + assert self.pp_handler is not None + all_decode_next = self.pp_handler.receive(input_batch) + # Optimistically update num_computed_tokens for entire batch here. + # Will be adjusted for rejections if necessary in update_requests. + self.postprocess_num_computed_tokens(input_batch) + if not all_decode_next: + # Might contain non-final prefill chunks, which will be scheduled + # in the immediate next step (rather than in pp_size steps). + self.model_state.postprocess_state(input_batch.idx_mapping, 0) # Post-step KV connector related operations. kv_connector_output = self.kv_connector.post_forward(finished_req_ids) @@ -1295,9 +1344,14 @@ class GPUModelRunner(LoRAModelRunnerMixin): hidden_states, input_batch, grammar_output ) - if self.use_pp: + if self.pp_handler is not None: # Broadcast to non-last PP ranks (handles spec decode multi-token). - pp_broadcast(sampler_output.sampled_token_ids, num_sampled, num_rejected) + self.pp_handler.broadcast( + sampler_output.sampled_token_ids, + num_sampled, + num_rejected, + input_batch, + ) assert self.prompt_logprobs_worker is not None prompt_logprobs_dict = self.prompt_logprobs_worker.compute_prompt_logprobs( @@ -1307,8 +1361,6 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.req_states.all_token_ids.gpu, self.req_states.num_computed_tokens.gpu, self.req_states.prompt_len.np, - self.req_states.prefill_len.np, - self.req_states.num_computed_prefill_tokens, ) # Prepare the model runner output. @@ -1334,17 +1386,14 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Get cached multimodal embeddings for draft forward. # NOTE: This is done here because postprocess updates # num_computed_prefill_tokens. - prefill_lens = self.req_states.prefill_len.np[input_batch.idx_mapping_np] - computed_prefill_lens = self.req_states.num_computed_prefill_tokens[ - input_batch.idx_mapping_np - ] mm_inputs = self.model_state.encoder_runner.gather_mm_embeddings( input_batch.req_ids, input_batch.num_tokens, input_batch.num_scheduled_tokens, input_batch.query_start_loc_np, - prefill_lens, - computed_prefill_lens + 1, # +1 to consider the skew in eagle + input_batch.prefill_len_np, + # +1 to consider the skew in eagle + input_batch.num_computed_prefill_tokens_np + 1, ) # Postprocess results and update request states. @@ -1352,8 +1401,12 @@ class GPUModelRunner(LoRAModelRunnerMixin): # ensuring that `copy_event` is recorded before calling postprocess. # This sequencing may slightly reduce latency as async D2H copy does not # need to wait for the postprocess to finish. - self.postprocess( - input_batch, sampler_output.sampled_token_ids, num_sampled, num_rejected + self.postprocess_sampled( + input_batch.idx_mapping, + sampler_output.sampled_token_ids, + num_sampled, + num_rejected, + input_batch.query_start_loc, ) if self.speculator is not None: @@ -1410,7 +1463,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): kv_connector_output = self.kv_connector.post_forward(finished_req_ids) if not self.is_last_pp_rank: - self.postprocess_pool(input_batch) + self.postprocess_num_computed_tokens(input_batch) return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output) assert self.pooling_runner is not None @@ -1432,14 +1485,14 @@ class GPUModelRunner(LoRAModelRunnerMixin): copy_stream=self.output_copy_stream, ) - self.postprocess_pool(input_batch) + self.postprocess_num_computed_tokens(input_batch) if self.use_async_scheduling: return async_output return async_output.get_output() - def postprocess_pool(self, input_batch: InputBatch) -> None: + def postprocess_num_computed_tokens(self, input_batch: InputBatch) -> None: # Update the number of computed tokens. - post_update_pool( + post_update_num_computed_tokens( input_batch.idx_mapping, self.req_states.num_computed_tokens.gpu, input_batch.query_start_loc, diff --git a/vllm/v1/worker/gpu/model_states/default.py b/vllm/v1/worker/gpu/model_states/default.py index 9f78749e78..ee5d9384fa 100644 --- a/vllm/v1/worker/gpu/model_states/default.py +++ b/vllm/v1/worker/gpu/model_states/default.py @@ -102,7 +102,6 @@ class DefaultModelState(ModelState): self, scheduled_encoder_inputs: dict[str, list[int]], input_batch: InputBatch, - req_states: RequestState, ) -> torch.Tensor: mm_hashes, mm_kwargs = self.encoder_runner.prepare_mm_inputs( scheduled_encoder_inputs @@ -118,8 +117,8 @@ class DefaultModelState(ModelState): input_batch.num_tokens, input_batch.num_scheduled_tokens, input_batch.query_start_loc_np, - req_states.prefill_len.np[input_batch.idx_mapping_np], - req_states.num_computed_prefill_tokens[input_batch.idx_mapping_np], + input_batch.prefill_len_np, + input_batch.num_computed_prefill_tokens_np, ) # Use unpadded input_ids to match is_mm_embed size (num_tokens). # input_batch.input_ids may be padded for CUDA graphs. diff --git a/vllm/v1/worker/gpu/model_states/interface.py b/vllm/v1/worker/gpu/model_states/interface.py index 721e5c2013..55bf8d473c 100644 --- a/vllm/v1/worker/gpu/model_states/interface.py +++ b/vllm/v1/worker/gpu/model_states/interface.py @@ -57,18 +57,13 @@ class ModelState(ABC): return None def postprocess_state( - self, - input_batch: InputBatch, - num_sampled: torch.Tensor, + self, idx_mapping: torch.Tensor, num_sampled: torch.Tensor ) -> None: return None @abstractmethod def get_mm_embeddings( - self, - scheduled_encoder_inputs: dict[str, list[int]], - input_batch: InputBatch, - req_states: RequestState, + self, scheduled_encoder_inputs: dict[str, list[int]], input_batch: InputBatch ) -> torch.Tensor | None: raise NotImplementedError diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py index b5abf56fbf..ced97c4f27 100644 --- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py @@ -9,6 +9,7 @@ import torch.nn as nn from vllm.config import VllmConfig from vllm.config.compilation import CUDAGraphMode +from vllm.triton_utils import tl, triton from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder from vllm.v1.kv_cache_interface import KVCacheConfig @@ -112,13 +113,10 @@ class MambaHybridModelState(DefaultModelState): # need the -1 sentinel rather than a raw zero draft count. num_decode_draft_tokens_np = np.full(num_reqs, -1, dtype=np.int32) if input_batch.num_draft_tokens_per_req is not None: - spec_decode_mask = ( - input_batch.num_draft_tokens_per_req > 0 - ) & ~input_batch.is_prefilling_np + has_draft_tokens = input_batch.num_draft_tokens_per_req > 0 + spec_decode_mask = has_draft_tokens & ~input_batch.is_prefilling_np num_decode_draft_tokens_np[: input_batch.num_reqs] = np.where( - spec_decode_mask, - input_batch.num_draft_tokens_per_req, - -1, + spec_decode_mask, input_batch.num_draft_tokens_per_req, -1 ) num_decode_draft_tokens_cpu = torch.from_numpy(num_decode_draft_tokens_np) @@ -145,12 +143,33 @@ class MambaHybridModelState(DefaultModelState): ) def postprocess_state( - self, - input_batch: InputBatch, - num_sampled: torch.Tensor, + self, idx_mapping: torch.Tensor, num_sampled: torch.Tensor | int ) -> None: # Chunked prefill does not sample a token, so num_sampled can be 0. # Mamba treats num_accepted_tokens=1 as the neutral non-spec value. - self.num_accepted_tokens_gpu[input_batch.idx_mapping] = torch.clamp( - num_sampled, min=1 - ) + if not isinstance(num_sampled, int): + # idx_mapping may contain -1 sentinels (filtered rows) under PP; the + # kernel skips them rather than scattering with a host-side gather. + num_reqs = idx_mapping.shape[0] + if num_reqs: + _scatter_num_accepted_kernel[(num_reqs,)]( + idx_mapping, num_sampled, self.num_accepted_tokens_gpu + ) + return + + # Fill with single value. + self.num_accepted_tokens_gpu.index_fill_(0, idx_mapping, max(num_sampled, 1)) + + +@triton.jit +def _scatter_num_accepted_kernel( + idx_mapping_ptr, # [num_reqs] batch_idx -> req_state_idx (-1 to skip) + num_sampled_ptr, # [num_reqs] + num_accepted_ptr, # [max_num_reqs] +): + row = tl.program_id(0) + req_state_idx = tl.load(idx_mapping_ptr + row) + if req_state_idx < 0: + return + num_sampled = tl.load(num_sampled_ptr + row) + tl.store(num_accepted_ptr + req_state_idx, tl.maximum(num_sampled, 1)) diff --git a/vllm/v1/worker/gpu/model_states/whisper.py b/vllm/v1/worker/gpu/model_states/whisper.py index 0ef3cadc87..b38cdae903 100644 --- a/vllm/v1/worker/gpu/model_states/whisper.py +++ b/vllm/v1/worker/gpu/model_states/whisper.py @@ -84,10 +84,7 @@ class WhisperModelState(ModelState): return ("transcription",) def get_mm_embeddings( - self, - scheduled_encoder_inputs: dict[str, list[int]], - input_batch: InputBatch, - req_states: RequestState, + self, scheduled_encoder_inputs: dict[str, list[int]], input_batch: InputBatch ) -> None: # Ensure encoder inputs are ordered consistently with input_batch.req_ids. encoder_inputs: dict[str, list[int]] = {} diff --git a/vllm/v1/worker/gpu/pp_utils.py b/vllm/v1/worker/gpu/pp_utils.py index bf379b5fb5..9f5d4c2d80 100644 --- a/vllm/v1/worker/gpu/pp_utils.py +++ b/vllm/v1/worker/gpu/pp_utils.py @@ -2,40 +2,193 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Pipeline Parallelism utils for V2 Model Runner.""" +from collections import deque +from dataclasses import dataclass + +import numpy as np import torch from vllm.distributed.parallel_state import get_pp_group +from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu +from vllm.v1.worker.gpu.input_batch import InputBatch -def pp_broadcast( - sampled_token_ids: torch.Tensor, - num_sampled: torch.Tensor, - num_rejected: torch.Tensor, -) -> None: - pp = get_pp_group() - assert pp.is_last_rank +@dataclass +class PendingRecv: + """Per-step slot data for a deferred postprocess on the main stream.""" - assert sampled_token_ids.dtype == torch.int64 - torch.distributed.broadcast( - sampled_token_ids.contiguous(), src=pp.last_rank, group=pp.device_group - ) + event: torch.cuda.Event - combined = torch.stack((num_sampled, num_rejected), dim=0) - torch.distributed.broadcast(combined, src=pp.last_rank, group=pp.device_group) + sampled_tokens: torch.Tensor # [num_reqs, max_sample_len] + num_sampled: torch.Tensor # [num_reqs] + num_rejected: torch.Tensor # [num_reqs] + idx_mapping: torch.Tensor # [num_reqs] + idx_mapping_np: np.ndarray # [num_reqs] + # Records which rows need a deferred postprocess (bool). + need_sampled_mask: np.ndarray # [num_reqs] + # Snapshot of slot generation counters at receive time, used to + # detect requests aborted since then. + gen_at_receive_np: np.ndarray # [num_reqs] -def pp_receive( - num_reqs: int, max_sample_len: int = 1 -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - pp = get_pp_group() - assert not pp.is_last_rank +def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None: + """Return a bool array of shape `[input_batch.num_reqs]` marking requests + with outputs that might be needed in a subsequent (decode) step. + Returns None if no sampled outputs are needed in the requests' next step.""" - sampled_tokens = torch.empty( - num_reqs, max_sample_len, dtype=torch.int64, device=pp.device - ) - torch.distributed.broadcast(sampled_tokens, src=pp.last_rank, group=pp.device_group) + old_computed = input_batch.num_computed_tokens_np + prefill_len = input_batch.prefill_len_np + max_seq_len = input_batch.max_seq_len_np + assert max_seq_len is not None # always populated under PP + # Exclude non-final prefill chunks (they don't produce a sample). + produces_sample = old_computed + input_batch.num_scheduled_tokens >= prefill_len + # Exclude requests that we know are finished. + not_finishing = np.maximum(old_computed, prefill_len) + 1 < max_seq_len + need_sampled_mask = produces_sample & not_finishing + return need_sampled_mask if need_sampled_mask.any() else None - combined = torch.empty(2, num_reqs, dtype=torch.int32, device=pp.device) - torch.distributed.broadcast(combined, src=pp.last_rank, group=pp.device_group) - num_sampled, num_rejected = combined.unbind(dim=0) - return sampled_tokens, num_sampled, num_rejected + +class PPHandler: + """Runs the PP sampled-token broadcast/recv on a side stream so the + default stream isn't gated by the matching peer call. Step T's recv is + consumed at step T+pp_size via `get_prev_sampled_outputs`. + + Uses a dedicated NCCL communicator (sibling of the PP `device_group`) + for the broadcast so it does not serialize on the wire with the + inter-stage hidden-state p2p send/recv ops. + """ + + def __init__( + self, max_num_reqs: int, num_speculative_steps: int, device: torch.device + ): + self.is_last_rank = get_pp_group().is_last_rank + self.last_rank = get_pp_group().last_rank + self.max_sample_len = num_speculative_steps + 1 + self.device = device + self.main_stream = torch.cuda.current_stream(device) + self.broadcast_stream = torch.cuda.Stream(device) + + # On non-last ranks, a FIFO with one entry per in-flight step: the entry + # pushed by step T's `receive` is consumed pp_size steps later. Pre-seeded + # with pp_size None placeholders so the first pp_size consumes are no-ops. + # None means no postprocess is pending for that step (broadcast skipped). + self.queue: deque[PendingRecv | None] = ( + deque() if self.is_last_rank else deque([None] * get_pp_group().world_size) + ) + + # Per req-index generation counter, incremented every time a request + # index is freed in RequestStats. Used for invalidating freed req data + # between PP decodes. + self.req_idx_gen_np = np.zeros(max_num_reqs, dtype=np.int32) + + # Dedicated subgroup for the sampled-token broadcast. + self.broadcast_group = get_pp_group().make_sibling_device_group( + group_desc="pp_broadcast" + ) + + def on_req_idx_freed(self, req_idx: int) -> None: + self.req_idx_gen_np[req_idx] += 1 + + def get_prev_sampled_outputs(self) -> dict[str, torch.Tensor] | None: + """Consume the entry from pp_size steps ago and wait for its recv event, + then filter out entries whose request was freed since `receive`. + """ + if not self.queue: + return None + slot = self.queue.popleft() + # Reserve this step's slot; `receive` overwrites it if applicable. + self.queue.append(None) + if slot is None: + return None + + # Skip requests which did not need sampled output and/or those already + # finished. The post_update kernel skips the -1 entries. + freed = self.req_idx_gen_np[slot.idx_mapping_np] != slot.gen_at_receive_np + exclude_mask = freed | ~slot.need_sampled_mask + idx_mapping = slot.idx_mapping + if exclude_mask.any(): + if exclude_mask.all(): + # No states require update anymore. + return None + # Filter excluded request indices. + idx_mapping_np = np.where(exclude_mask, -1, slot.idx_mapping_np) + idx_mapping = async_copy_to_gpu(idx_mapping_np, device=self.device) + + self.main_stream.wait_event(slot.event) + return dict( + sampled_tokens=slot.sampled_tokens, + num_sampled=slot.num_sampled, + num_rejected=slot.num_rejected, + idx_mapping=idx_mapping, + ) + + def receive(self, input_batch: InputBatch) -> bool: + """Returns True iff sampled tokens need to be gathered from *all* + requests in the batch.""" + assert not self.is_last_rank + need_sampled_mask = compute_need_sampled_mask(input_batch) + if need_sampled_mask is None: + # Leave this step's reserved slot as None. + return False + + # Snapshot the per-slot generation counter so a later free of any of + # these RequestStates request indices is detectable at consume time. + gen_at_receive_np = self.req_idx_gen_np[input_batch.idx_mapping_np] + + num_reqs = input_batch.num_reqs + with torch.cuda.stream(self.broadcast_stream): + self.broadcast_stream.wait_stream(self.main_stream) + sampled_tokens = torch.empty( + num_reqs, self.max_sample_len, dtype=torch.int64, device=self.device + ) + combined = torch.empty(2, num_reqs, dtype=torch.int32, device=self.device) + torch.distributed.broadcast( + sampled_tokens, src=self.last_rank, group=self.broadcast_group + ) + torch.distributed.broadcast( + combined, src=self.last_rank, group=self.broadcast_group + ) + event = self.broadcast_stream.record_event() + num_sampled, num_rejected = combined.unbind(dim=0) + # Must record_stream since these were allocated on broadcast stream but + # later used on the main stream. + sampled_tokens.record_stream(self.main_stream) + combined.record_stream(self.main_stream) + self.queue[-1] = PendingRecv( + event, + sampled_tokens, + num_sampled, + num_rejected, + input_batch.idx_mapping, + input_batch.idx_mapping_np, + need_sampled_mask, + gen_at_receive_np, + ) + return bool(need_sampled_mask.all()) + + def broadcast( + self, + sampled_token_ids: torch.Tensor, + num_sampled: torch.Tensor, + num_rejected: torch.Tensor, + input_batch: InputBatch, + ) -> None: + assert self.is_last_rank + if compute_need_sampled_mask(input_batch) is None: + # No request needs sampled outputs for a subsequent decode step. + return + + assert sampled_token_ids.dtype == torch.int64 + with torch.cuda.stream(self.broadcast_stream): + self.broadcast_stream.wait_stream(self.main_stream) + torch.distributed.broadcast( + sampled_token_ids.contiguous(), + src=self.last_rank, + group=self.broadcast_group, + ) + combined = torch.stack((num_sampled, num_rejected), dim=0) + torch.distributed.broadcast( + combined, src=self.last_rank, group=self.broadcast_group + ) + for tensor in (sampled_token_ids, num_sampled, num_rejected): + tensor.record_stream(self.broadcast_stream) diff --git a/vllm/v1/worker/gpu/sample/prompt_logprob.py b/vllm/v1/worker/gpu/sample/prompt_logprob.py index 71feb7cf0e..b89ebac35d 100644 --- a/vllm/v1/worker/gpu/sample/prompt_logprob.py +++ b/vllm/v1/worker/gpu/sample/prompt_logprob.py @@ -42,10 +42,6 @@ class PromptLogprobsWorker: num_computed_tokens: torch.Tensor, # [max_num_reqs] prompt_lens: np.ndarray, - # [max_num_reqs] - prefill_lens: np.ndarray, - # [max_num_reqs] - num_computed_prefill_tokens: np.ndarray, ) -> dict[str, LogprobsTensors]: idx_mapping_np = input_batch.idx_mapping_np needs_prompt_logprobs = self.uses_prompt_logprobs[idx_mapping_np] @@ -55,11 +51,11 @@ class PromptLogprobsWorker: num_prompt_logprobs = self.num_prompt_logprobs[idx_mapping_np] prompt_lens = prompt_lens[idx_mapping_np] - computed_prefill = num_computed_prefill_tokens[idx_mapping_np] + computed_prefill = input_batch.num_computed_prefill_tokens_np includes_prompt = computed_prefill < prompt_lens # NOTE(woosuk): If the request was resumed after preemption, its prompt # logprobs must have been computed before preemption. Skip. - resumed_after_prompt = prompt_lens < prefill_lens[idx_mapping_np] + resumed_after_prompt = prompt_lens < input_batch.prefill_len_np needs_prompt_logprobs &= includes_prompt & ~resumed_after_prompt if not np.any(needs_prompt_logprobs): return {} diff --git a/vllm/v1/worker/gpu/states.py b/vllm/v1/worker/gpu/states.py index cdd7286fa5..be7bae7f17 100644 --- a/vllm/v1/worker/gpu/states.py +++ b/vllm/v1/worker/gpu/states.py @@ -65,6 +65,9 @@ class RequestState: self.max_num_reqs, 1, dtype=torch.int64, device=device ) + # Max total seq length (prompt_len + max_tokens). + self.max_seq_len = np.zeros(self.max_num_reqs, dtype=np.int32) + # Draft tokens. self.draft_tokens = torch.zeros( self.max_num_reqs, @@ -87,12 +90,14 @@ class RequestState: prompt_len: int, all_token_ids: list[int], num_computed_tokens: int, + max_tokens: int, ) -> None: assert len(self.free_indices) > 0, "No free indices" req_idx = self.free_indices.pop() self.req_id_to_index[req_id] = req_idx self.index_to_req_id[req_idx] = req_id + self.max_seq_len[req_idx] = prompt_len + max_tokens self.prompt_len.np[req_idx] = prompt_len prefill_len = len(all_token_ids) assert prefill_len >= prompt_len, ( @@ -124,17 +129,11 @@ class RequestState: self.all_token_ids.apply_write() self.num_computed_tokens.apply_write() - def remove_request(self, req_id: str) -> bool: + def remove_request(self, req_id: str) -> int | None: + """Return the freed slot index, or None if the request was not found.""" req_idx = self.req_id_to_index.pop(req_id, None) if req_idx is None: - # Request not found. - return False + return None self.index_to_req_id.pop(req_idx, None) self.free_indices.append(req_idx) - return True - - def is_prefilling(self, idx_mapping_np: np.ndarray) -> np.ndarray: - return ( - self.num_computed_prefill_tokens[idx_mapping_np] - < self.prefill_len.np[idx_mapping_np] - ) + return req_idx