[ModelRunnerV2] Avoid pipeline parallel bubbles (#42187)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -94,11 +94,13 @@ steps:
|
||||
- vllm/v1/worker/gpu_worker.py
|
||||
- tests/distributed/test_pipeline_parallel.py
|
||||
- tests/distributed/test_pp_cudagraph.py
|
||||
- tests/v1/distributed/test_pp_dp_v2.py
|
||||
commands:
|
||||
- set -x
|
||||
- export VLLM_USE_V2_MODEL_RUNNER=1
|
||||
- pytest -v -s distributed/test_pipeline_parallel.py -k "not ray and not Jamba"
|
||||
- pytest -v -s distributed/test_pp_cudagraph.py -k "not ray"
|
||||
- pytest -v -s v1/distributed/test_pp_dp_v2.py
|
||||
|
||||
- label: Model Runner V2 Spec Decode
|
||||
device: h200_35gb
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""V2 ModelRunner + pipeline parallel + data parallel integration tests.
|
||||
|
||||
Covers the interaction between the V2 model runner's PP sampled-token
|
||||
broadcast and the DP per-step all-reduce across a few concurrency
|
||||
regimes. Requires 4 GPUs (DP=2, PP=2, TP=1) on CUDA.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import os
|
||||
from contextlib import ExitStack
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
PP_DP_MODEL = "ibm-research/PowerMoE-3b" # smallest cached MoE that supports PP
|
||||
PROMPT = "This is a test of data parallel and pipeline parallel together"
|
||||
|
||||
|
||||
def _gpu_skip_reason() -> str | None:
|
||||
if not current_platform.is_cuda():
|
||||
return "requires CUDA"
|
||||
n = current_platform.device_count()
|
||||
if n < 4:
|
||||
return f"requires 4 GPUs, got {n}"
|
||||
return None
|
||||
|
||||
|
||||
_GPU_SKIP = _gpu_skip_reason()
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skipif(
|
||||
os.environ.get("VLLM_USE_V2_MODEL_RUNNER", "0") != "1",
|
||||
reason="VLLM_USE_V2_MODEL_RUNNER=1 required",
|
||||
),
|
||||
pytest.mark.skipif(_GPU_SKIP is not None, reason=_GPU_SKIP or ""),
|
||||
]
|
||||
|
||||
|
||||
def _engine_args(async_scheduling: bool) -> AsyncEngineArgs:
|
||||
return AsyncEngineArgs(
|
||||
model=PP_DP_MODEL,
|
||||
pipeline_parallel_size=2,
|
||||
data_parallel_size=2,
|
||||
data_parallel_backend="mp",
|
||||
tensor_parallel_size=1,
|
||||
max_model_len=4096,
|
||||
max_num_batched_tokens=2048,
|
||||
max_num_seqs=256,
|
||||
async_scheduling=async_scheduling,
|
||||
enable_prefix_caching=False,
|
||||
enforce_eager=False,
|
||||
enable_expert_parallel=False,
|
||||
)
|
||||
|
||||
|
||||
async def _generate(engine: AsyncLLM, prompt: str, max_tokens: int) -> int:
|
||||
"""Run one streaming completion and return the number of tokens it yielded."""
|
||||
sampling_params = SamplingParams(
|
||||
max_tokens=max_tokens,
|
||||
ignore_eos=True,
|
||||
output_kind=RequestOutputKind.DELTA,
|
||||
temperature=0.0,
|
||||
)
|
||||
request_id = f"req-{id(prompt):x}-{max_tokens}"
|
||||
total = 0
|
||||
async for out in engine.generate(
|
||||
request_id=request_id, prompt=prompt, sampling_params=sampling_params
|
||||
):
|
||||
total += len(out.outputs[0].token_ids)
|
||||
return total
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
async def test_pp_dp_v2_low_concurrency(async_scheduling: bool):
|
||||
"""A single in-flight request at a time, repeated, to exercise the
|
||||
PP slot ring under empty batches between decodes."""
|
||||
with ExitStack() as after:
|
||||
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling))
|
||||
after.callback(engine.shutdown)
|
||||
|
||||
for _ in range(4):
|
||||
n = await _generate(engine, PROMPT, max_tokens=16)
|
||||
assert n == 16
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
async def test_pp_dp_v2_mid_concurrency(async_scheduling: bool):
|
||||
"""64 concurrent requests, staggered, to exercise the steady-state
|
||||
DP all-reduce + PP slot-ring path."""
|
||||
with ExitStack() as after:
|
||||
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling))
|
||||
after.callback(engine.shutdown)
|
||||
|
||||
async def _one(i: int) -> int:
|
||||
await asyncio.sleep(0.01 * i) # stagger so DP load-balances
|
||||
return await _generate(engine, f"{PROMPT} {i}", max_tokens=64)
|
||||
|
||||
results = await asyncio.gather(*[_one(i) for i in range(64)])
|
||||
assert all(n == 64 for n in results), results
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pp_dp_v2_abort_mid_decode():
|
||||
"""Cancel half the in-flight requests mid-stream and confirm the
|
||||
engine survives the abort storm."""
|
||||
|
||||
with ExitStack() as after:
|
||||
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling=True))
|
||||
after.callback(engine.shutdown)
|
||||
|
||||
async def _maybe_cancel(i: int):
|
||||
sampling_params = SamplingParams(
|
||||
max_tokens=64,
|
||||
ignore_eos=True,
|
||||
output_kind=RequestOutputKind.DELTA,
|
||||
temperature=0.0,
|
||||
)
|
||||
request_id = f"abort-req-{i}"
|
||||
count = 0
|
||||
cancel_at = 4 if i % 2 == 0 else 64
|
||||
async for out in engine.generate(
|
||||
request_id=request_id,
|
||||
prompt=f"{PROMPT} {i}",
|
||||
sampling_params=sampling_params,
|
||||
):
|
||||
count += len(out.outputs[0].token_ids)
|
||||
if count >= cancel_at:
|
||||
break
|
||||
return count, i
|
||||
|
||||
results = await asyncio.gather(*[_maybe_cancel(i) for i in range(32)])
|
||||
for count, i in results:
|
||||
if i % 2 == 0:
|
||||
assert count >= 4
|
||||
else:
|
||||
assert count == 64
|
||||
|
||||
# Engine must still serve after the abort storm.
|
||||
final = await _generate(engine, "post-abort warmup", max_tokens=8)
|
||||
assert final == 8
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pp_dp_v2_pause_resume():
|
||||
"""Pause an engine with a request in flight, then resume and confirm
|
||||
new requests still work."""
|
||||
|
||||
with ExitStack() as after:
|
||||
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling=True))
|
||||
after.callback(engine.shutdown)
|
||||
|
||||
# Start a long-running generation, let some decoding happen, then
|
||||
# pause (abort mode) and confirm the in-flight task terminates.
|
||||
inflight = asyncio.create_task(_generate(engine, PROMPT, max_tokens=128))
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
assert not await engine.is_paused()
|
||||
await engine.pause_generation(mode="abort")
|
||||
assert await engine.is_paused()
|
||||
|
||||
with contextlib.suppress(Exception):
|
||||
await inflight
|
||||
|
||||
await engine.resume_generation()
|
||||
assert not await engine.is_paused()
|
||||
|
||||
n = await _generate(engine, PROMPT, max_tokens=8)
|
||||
assert n == 8
|
||||
@@ -167,7 +167,9 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
|
||||
events = []
|
||||
runner = _make_runner(is_last_pp_rank=False, num_speculative_steps=0)
|
||||
runner.execute_model_state = SimpleNamespace(
|
||||
input_batch=SimpleNamespace(num_reqs=2),
|
||||
input_batch=SimpleNamespace(
|
||||
num_reqs=2, idx_mapping=torch.zeros(2, dtype=torch.int32)
|
||||
),
|
||||
attn_metadata=None,
|
||||
slot_mappings_by_layer=None,
|
||||
hidden_states=None,
|
||||
@@ -175,18 +177,19 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
|
||||
finished_req_ids=set(),
|
||||
num_tokens_across_dp=None,
|
||||
)
|
||||
runner.postprocess = lambda *args, **kwargs: events.append("postprocess")
|
||||
runner.eplb.step = lambda *args, **kwargs: events.append("eplb")
|
||||
monkeypatch.setattr(
|
||||
mrv2,
|
||||
"pp_receive",
|
||||
lambda *args, **kwargs: (
|
||||
torch.zeros((2, 1), dtype=torch.long),
|
||||
torch.ones(2, dtype=torch.int32),
|
||||
torch.zeros(2, dtype=torch.int32),
|
||||
),
|
||||
runner.req_states = SimpleNamespace()
|
||||
|
||||
def fake_receive(*args, **kwargs):
|
||||
events.append("receive")
|
||||
# all_decode_next=True, so model_state.postprocess_state is skipped.
|
||||
return True
|
||||
|
||||
runner.pp_handler = SimpleNamespace(receive=fake_receive)
|
||||
runner.postprocess_num_computed_tokens = lambda *args, **kwargs: events.append(
|
||||
"postprocess_num_computed_tokens"
|
||||
)
|
||||
runner.eplb.step = lambda *args, **kwargs: events.append("eplb")
|
||||
|
||||
output = mrv2.GPUModelRunner.sample_tokens(runner, None)
|
||||
assert output in (EMPTY_MODEL_RUNNER_OUTPUT, None)
|
||||
assert events == ["postprocess", "eplb"]
|
||||
assert events == ["receive", "postprocess_num_computed_tokens", "eplb"]
|
||||
|
||||
+7
-3
@@ -498,9 +498,13 @@ class VllmConfig:
|
||||
# PP requires PP-size concurrent batches to fill the pipeline.
|
||||
# Async scheduling requires 2 concurrent batches to overlap.
|
||||
pp_size = self.parallel_config.pipeline_parallel_size
|
||||
if pp_size > 1:
|
||||
return pp_size
|
||||
return 2 if self.scheduler_config.async_scheduling else 1
|
||||
if self.scheduler_config.async_scheduling:
|
||||
if self.use_v2_model_runner:
|
||||
return pp_size + 1
|
||||
# V1 Model Runner does not fully support async scheduling with PP.
|
||||
if pp_size <= 1:
|
||||
return 2
|
||||
return pp_size
|
||||
|
||||
@property
|
||||
def num_speculative_tokens(self) -> int:
|
||||
|
||||
@@ -359,6 +359,9 @@ class GroupCoordinator:
|
||||
assert self_cpu_group is not None
|
||||
assert self_device_group is not None
|
||||
|
||||
self.group_ranks = group_ranks
|
||||
self.torch_distributed_backend = torch_distributed_backend
|
||||
|
||||
self.cpu_group = self_cpu_group
|
||||
self.device_group = self_device_group
|
||||
|
||||
@@ -406,6 +409,22 @@ class GroupCoordinator:
|
||||
and getattr(self.device_communicator, "supports_tensor_dict", False)
|
||||
)
|
||||
|
||||
def make_sibling_device_group(self, group_desc: str | None = None) -> ProcessGroup:
|
||||
"""Create a new device-side ProcessGroup with the same per-rank membership
|
||||
as this coordinator's `device_group`, but backed by a distinct communicator.
|
||||
This is a collective call: every world rank must invoke it. Used where we
|
||||
want to issue ops that can run concurrently with ops on `device_group`.
|
||||
"""
|
||||
sibling: ProcessGroup | None = None
|
||||
for ranks in self.group_ranks:
|
||||
pg = torch.distributed.new_group(
|
||||
ranks, backend=self.torch_distributed_backend, group_desc=group_desc
|
||||
)
|
||||
if self.rank in ranks:
|
||||
sibling = pg
|
||||
assert sibling is not None
|
||||
return sibling
|
||||
|
||||
def create_mq_broadcaster(
|
||||
self, writer_rank=0, external_writer_handle=None, blocking=True
|
||||
):
|
||||
|
||||
@@ -14,6 +14,7 @@ class AsyncScheduler(Scheduler):
|
||||
super().__init__(*args, **kwargs)
|
||||
# reusable read-only placeholder list for speculative decoding.
|
||||
self._spec_token_placeholders: list[int] = [-1] * self.num_spec_tokens
|
||||
self.pp_size = self.parallel_config.pipeline_parallel_size
|
||||
|
||||
def _update_after_schedule(self, scheduler_output: SchedulerOutput) -> None:
|
||||
super()._update_after_schedule(scheduler_output)
|
||||
@@ -34,6 +35,11 @@ class AsyncScheduler(Scheduler):
|
||||
# We will update the actual spec token ids in the worker process.
|
||||
request.spec_token_ids = self._spec_token_placeholders
|
||||
|
||||
if self.use_v2_model_runner:
|
||||
# Set the next step index in which this request is eligible to be
|
||||
# scheduled for decode (for PP microbatching).
|
||||
request.next_decode_eligible_step = self.current_step + self.pp_size
|
||||
|
||||
def _update_request_with_output(
|
||||
self, request: Request, new_token_ids: list[int]
|
||||
) -> tuple[list[int], bool]:
|
||||
|
||||
@@ -248,6 +248,9 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
self.use_pp = self.parallel_config.pipeline_parallel_size > 1
|
||||
self.use_v2_model_runner = vllm_config.use_v2_model_runner
|
||||
# Scheduler iteration counter. Drives the V2+PP+async decode-throttle
|
||||
# cadence (`next_decode_eligible_step`).
|
||||
self.current_step = 0
|
||||
self.scheduler_reserve_full_isl = (
|
||||
self.scheduler_config.scheduler_reserve_full_isl
|
||||
)
|
||||
@@ -333,6 +336,7 @@ class Scheduler(SchedulerInterface):
|
||||
return num_new_tokens
|
||||
|
||||
def schedule(self) -> SchedulerOutput:
|
||||
self.current_step += 1
|
||||
# NOTE(woosuk) on the scheduling algorithm:
|
||||
# There's no "decoding phase" nor "prefill phase" in the scheduler.
|
||||
# Each request just has the num_computed_tokens and
|
||||
@@ -388,6 +392,12 @@ class Scheduler(SchedulerInterface):
|
||||
req_index += 1
|
||||
continue
|
||||
|
||||
if self.current_step < request.next_decode_eligible_step:
|
||||
# V2+PP+async: enforce `pp_size` steps between same-req decodes
|
||||
# to match worker-side sampled-tokens broadcast slot ring cadence.
|
||||
req_index += 1
|
||||
continue
|
||||
|
||||
num_new_tokens = (
|
||||
request.num_tokens_with_spec
|
||||
+ request.num_output_placeholders
|
||||
|
||||
@@ -534,14 +534,12 @@ class EngineCore:
|
||||
if not deferred_scheduler_output:
|
||||
# Add this step's future to the queue.
|
||||
batch_queue.appendleft((future, scheduler_output, exec_future))
|
||||
if (
|
||||
model_executed
|
||||
and len(batch_queue) < self.batch_queue_size
|
||||
and not batch_queue[-1][0].done()
|
||||
if len(batch_queue) < self.batch_queue_size and (
|
||||
model_executed or self.scheduler.has_requests()
|
||||
):
|
||||
# Don't block on next worker response unless the queue is full
|
||||
# or there are no more requests to schedule.
|
||||
return None, True
|
||||
return None, model_executed
|
||||
|
||||
elif not batch_queue:
|
||||
# Queue is empty. We should not reach here since this method should
|
||||
|
||||
@@ -141,6 +141,10 @@ class Request:
|
||||
self.num_output_placeholders = 0
|
||||
self.async_tokens_to_discard = 0
|
||||
|
||||
# V2+PP+async: Enforces `pp_size` cadence between same-request decode steps
|
||||
# so the worker's broadcast slot ring stays consistent.
|
||||
self.next_decode_eligible_step = 0
|
||||
|
||||
self.spec_token_ids: list[int] = []
|
||||
self.num_computed_tokens = 0
|
||||
self.cache_salt: str | None = cache_salt
|
||||
|
||||
@@ -13,6 +13,15 @@ from vllm.utils.torch_utils import (
|
||||
get_accelerator_view_from_cpu_tensor,
|
||||
)
|
||||
|
||||
# Default round-robin depth for the UVA buffer pools. Must be >= the number of
|
||||
# concurrent in-flight steps (engine batch_queue_size).
|
||||
_DEFAULT_MAX_CONCURRENCY = 2
|
||||
|
||||
|
||||
def set_default_max_concurrency(n: int) -> None:
|
||||
global _DEFAULT_MAX_CONCURRENCY
|
||||
_DEFAULT_MAX_CONCURRENCY = max(2, n)
|
||||
|
||||
|
||||
def async_copy_to_gpu(
|
||||
x: torch.Tensor | np.ndarray,
|
||||
@@ -47,8 +56,10 @@ class UvaBufferPool:
|
||||
self,
|
||||
size: int | Sequence[int],
|
||||
dtype: torch.dtype,
|
||||
max_concurrency: int = 2,
|
||||
max_concurrency: int | None = None,
|
||||
):
|
||||
if max_concurrency is None:
|
||||
max_concurrency = _DEFAULT_MAX_CONCURRENCY
|
||||
self.size = size
|
||||
self.dtype = dtype
|
||||
self.max_concurrency = max_concurrency
|
||||
@@ -80,7 +91,10 @@ class UvaBufferPool:
|
||||
|
||||
class UvaBackedTensor:
|
||||
def __init__(
|
||||
self, size: int | Sequence[int], dtype: torch.dtype, max_concurrency: int = 2
|
||||
self,
|
||||
size: int | Sequence[int],
|
||||
dtype: torch.dtype,
|
||||
max_concurrency: int | None = None,
|
||||
):
|
||||
self.dtype = dtype
|
||||
|
||||
@@ -104,9 +118,11 @@ class StagedWriteTensor:
|
||||
size: int | Sequence[int],
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
max_concurrency: int = 2,
|
||||
max_concurrency: int | None = None,
|
||||
uva_instead_of_gpu: bool = False,
|
||||
):
|
||||
if max_concurrency is None:
|
||||
max_concurrency = _DEFAULT_MAX_CONCURRENCY
|
||||
supported_dtypes = [torch.int32, torch.int64, torch.float32]
|
||||
if dtype not in supported_dtypes:
|
||||
raise ValueError(
|
||||
|
||||
@@ -67,9 +67,18 @@ class InputBatch:
|
||||
seq_lens_cpu_upper_bound: torch.Tensor
|
||||
# [num_reqs]
|
||||
dcp_local_seq_lens: torch.Tensor | None
|
||||
# [num_reqs] CPU bool array.
|
||||
# [num_reqs]
|
||||
num_computed_tokens_np: np.ndarray
|
||||
# [num_reqs]
|
||||
prefill_len_np: np.ndarray
|
||||
# [num_reqs]
|
||||
num_computed_prefill_tokens_np: np.ndarray
|
||||
# [num_reqs] CPU bool array == (num_computed_prefill_tokens_np < prefill_len_np).
|
||||
is_prefilling_np: np.ndarray
|
||||
|
||||
# [num_reqs] only populated when pipeline parallelism is enabled.
|
||||
max_seq_len_np: np.ndarray | None
|
||||
|
||||
# [num_tokens_after_padding]
|
||||
input_ids: torch.Tensor
|
||||
# [num_tokens_after_padding]
|
||||
@@ -148,7 +157,11 @@ class InputBatch:
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=None,
|
||||
num_computed_tokens_np=np.zeros(num_reqs, dtype=np.int32),
|
||||
prefill_len_np=np.zeros(num_reqs, dtype=np.int32),
|
||||
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
|
||||
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
|
||||
max_seq_len_np=None,
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
logits_indices=logits_indices,
|
||||
@@ -438,6 +451,9 @@ def _post_update_kernel(
|
||||
):
|
||||
req_id = tl.program_id(0)
|
||||
req_state_idx = tl.load(idx_mapping_ptr + req_id)
|
||||
if req_state_idx < 0:
|
||||
# Filter rows with negative index entries.
|
||||
return
|
||||
|
||||
total_len = tl.load(total_len_ptr + req_state_idx)
|
||||
num_sampled = tl.load(num_sampled_ptr + req_id)
|
||||
@@ -464,18 +480,22 @@ def _post_update_kernel(
|
||||
count = tl.load(token_ptr)
|
||||
tl.store(token_ptr, count + 1)
|
||||
|
||||
query_start = tl.load(query_start_loc_ptr + req_id)
|
||||
query_end = tl.load(query_start_loc_ptr + req_id + 1)
|
||||
query_len = query_end - query_start
|
||||
if query_start_loc_ptr is None:
|
||||
query_len = 0
|
||||
else:
|
||||
query_start = tl.load(query_start_loc_ptr + req_id)
|
||||
query_end = tl.load(query_start_loc_ptr + req_id + 1)
|
||||
query_len = query_end - query_start
|
||||
num_rejected = tl.load(num_rejected_ptr + req_id)
|
||||
|
||||
num_computed = tl.load(num_computed_tokens_ptr + req_state_idx)
|
||||
num_computed += query_len - num_rejected
|
||||
tl.store(num_computed_tokens_ptr + req_state_idx, num_computed)
|
||||
computed_delta = query_len - num_rejected
|
||||
if computed_delta != 0:
|
||||
num_computed = tl.load(num_computed_tokens_ptr + req_state_idx)
|
||||
tl.store(num_computed_tokens_ptr + req_state_idx, num_computed + computed_delta)
|
||||
|
||||
|
||||
def post_update(
|
||||
# [num_reqs]
|
||||
# [num_reqs] batch_idx -> req_state_idx; negative index means skip.
|
||||
idx_mapping: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
num_computed_tokens: torch.Tensor,
|
||||
@@ -490,7 +510,7 @@ def post_update(
|
||||
# [num_reqs]
|
||||
num_rejected: torch.Tensor,
|
||||
# [num_reqs + 1]
|
||||
query_start_loc: torch.Tensor,
|
||||
query_start_loc: torch.Tensor | None,
|
||||
# [max_num_reqs, max_model_len]
|
||||
all_token_ids: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
@@ -516,7 +536,7 @@ def post_update(
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _post_update_pool_kernel(
|
||||
def _post_update_num_computed_tokens_kernel(
|
||||
idx_mapping_ptr,
|
||||
num_computed_tokens_ptr,
|
||||
query_start_loc_ptr,
|
||||
@@ -531,7 +551,7 @@ def _post_update_pool_kernel(
|
||||
tl.store(num_computed_tokens_ptr + req_state_idx, num_computed + query_len)
|
||||
|
||||
|
||||
def post_update_pool(
|
||||
def post_update_num_computed_tokens(
|
||||
# [num_reqs]
|
||||
idx_mapping: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
@@ -540,7 +560,7 @@ def post_update_pool(
|
||||
query_start_loc: torch.Tensor,
|
||||
) -> None:
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
_post_update_pool_kernel[(num_reqs,)](
|
||||
_post_update_num_computed_tokens_kernel[(num_reqs,)](
|
||||
idx_mapping,
|
||||
num_computed_tokens,
|
||||
query_start_loc,
|
||||
|
||||
@@ -61,7 +61,10 @@ from vllm.v1.worker.gpu.attn_utils import (
|
||||
init_kv_cache,
|
||||
)
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
|
||||
from vllm.v1.worker.gpu.buffer_utils import (
|
||||
async_copy_to_gpu,
|
||||
set_default_max_concurrency,
|
||||
)
|
||||
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import (
|
||||
BatchExecutionDescriptor,
|
||||
@@ -77,7 +80,7 @@ from vllm.v1.worker.gpu.input_batch import (
|
||||
expand_idx_mapping,
|
||||
get_num_sampled_and_rejected,
|
||||
post_update,
|
||||
post_update_pool,
|
||||
post_update_num_computed_tokens,
|
||||
prepare_pos_seq_lens,
|
||||
prepare_prefill_inputs,
|
||||
)
|
||||
@@ -90,7 +93,7 @@ from vllm.v1.worker.gpu.lora_utils import LoraState
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.model_states import init_model_state
|
||||
from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner
|
||||
from vllm.v1.worker.gpu.pp_utils import pp_broadcast, pp_receive
|
||||
from vllm.v1.worker.gpu.pp_utils import PPHandler
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
from vllm.v1.worker.gpu.sample.prompt_logprob import PromptLogprobsWorker
|
||||
from vllm.v1.worker.gpu.sample.sampler import Sampler
|
||||
@@ -149,6 +152,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.is_first_pp_rank = get_pp_group().is_first_rank
|
||||
self.is_last_pp_rank = get_pp_group().is_last_rank
|
||||
|
||||
# Size the UVA buffer pools to the max number of concurrent in-flight
|
||||
# steps. Must run before any pooled buffer is constructed
|
||||
set_default_max_concurrency(vllm_config.max_concurrent_batches)
|
||||
|
||||
# PP broadcast/recv helper. Runs the collective on a side stream.
|
||||
self.pp_handler: PPHandler | None = None
|
||||
|
||||
# Persistent buffer for intermediate tensors (non-first PP ranks).
|
||||
self.intermediate_tensors: IntermediateTensors | None = None
|
||||
|
||||
@@ -210,6 +220,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
if self.use_pp:
|
||||
self.pp_handler = PPHandler(
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
num_speculative_steps=self.num_speculative_steps,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
self.sampler: Sampler | None = None
|
||||
self.rejection_sampler: RejectionSampler | None = None
|
||||
self.prompt_logprobs_worker: PromptLogprobsWorker | None = None
|
||||
@@ -688,8 +705,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
return cuda_graph_size
|
||||
|
||||
def _remove_request(self, req_id: str) -> bool:
|
||||
if not self.req_states.remove_request(req_id):
|
||||
req_idx = self.req_states.remove_request(req_id)
|
||||
if req_idx is None:
|
||||
return False
|
||||
if self.pp_handler is not None:
|
||||
self.pp_handler.on_req_idx_freed(req_idx)
|
||||
if self.encoder_cache is not None:
|
||||
self.encoder_cache.remove_request(req_id)
|
||||
if self.prompt_logprobs_worker is not None:
|
||||
@@ -710,6 +730,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
for mm_hash in scheduler_output.free_encoder_mm_hashes:
|
||||
self.encoder_cache.free_encoder_cache(mm_hash)
|
||||
|
||||
def update_pp_decode_requests(self):
|
||||
# For non-last PP ranks, update decode requests with sampler output from
|
||||
# the prior step in which they were scheduled (pp_size steps ago).
|
||||
if self.pp_handler is not None:
|
||||
outputs = self.pp_handler.get_prev_sampled_outputs()
|
||||
if outputs is not None:
|
||||
self.postprocess_sampled(**outputs)
|
||||
|
||||
def add_requests(self, scheduler_output: SchedulerOutput) -> None:
|
||||
for new_req_data in scheduler_output.scheduled_new_reqs:
|
||||
assert new_req_data.prompt_token_ids is not None
|
||||
@@ -722,11 +750,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self._remove_request(req_id)
|
||||
|
||||
prompt_len = len(new_req_data.prompt_token_ids)
|
||||
sampling_params = new_req_data.sampling_params
|
||||
self.req_states.add_request(
|
||||
req_id=req_id,
|
||||
prompt_len=prompt_len,
|
||||
all_token_ids=new_req_data.prefill_token_ids,
|
||||
num_computed_tokens=new_req_data.num_computed_tokens,
|
||||
max_tokens=sampling_params.max_tokens if sampling_params else 1, # type: ignore[arg-type]
|
||||
)
|
||||
req_index = self.req_states.req_id_to_index[req_id]
|
||||
|
||||
@@ -769,7 +799,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
req_index, req_new_block_ids, overwrite=False
|
||||
)
|
||||
|
||||
# Update num_computed_prefill_tokens.
|
||||
# Update CPU num_computed_prefill_tokens.
|
||||
np.minimum(
|
||||
self.req_states.num_computed_tokens_np,
|
||||
self.req_states.prefill_len.np,
|
||||
@@ -848,7 +878,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc)
|
||||
query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1]
|
||||
query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1]
|
||||
is_prefilling_np = self.req_states.is_prefilling(idx_mapping_np)
|
||||
prefill_len_np = self.req_states.prefill_len.np[idx_mapping_np]
|
||||
computed_prefill_tokens_np = self.req_states.num_computed_prefill_tokens
|
||||
num_computed_prefill_tokens_np = computed_prefill_tokens_np[idx_mapping_np]
|
||||
is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np
|
||||
|
||||
# Get prefill tokens if any.
|
||||
if np.any(is_prefilling_np):
|
||||
@@ -900,13 +933,19 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
|
||||
# CPU upper bound on seq_lens; padded entries left at zero.
|
||||
num_computed_tokens_np = self.req_states.num_computed_tokens_np[idx_mapping_np]
|
||||
seq_lens_cpu_upper_bound_np = np.zeros(num_reqs_padded, dtype=np.int32)
|
||||
np.add(
|
||||
self.req_states.num_computed_tokens_np[idx_mapping_np],
|
||||
num_computed_tokens_np,
|
||||
num_scheduled_tokens,
|
||||
out=seq_lens_cpu_upper_bound_np[:num_reqs],
|
||||
)
|
||||
seq_lens_cpu_upper_bound = torch.from_numpy(seq_lens_cpu_upper_bound_np)
|
||||
|
||||
max_seq_len_np = None
|
||||
if self.use_pp:
|
||||
# max_seq_len is only consumed by the PP `compute_need_sampled_mask`
|
||||
max_seq_len_np = self.req_states.max_seq_len[idx_mapping_np]
|
||||
return InputBatch(
|
||||
req_ids=req_ids,
|
||||
num_reqs=num_reqs,
|
||||
@@ -925,7 +964,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
num_computed_tokens_np=num_computed_tokens_np,
|
||||
prefill_len_np=prefill_len_np,
|
||||
num_computed_prefill_tokens_np=num_computed_prefill_tokens_np,
|
||||
is_prefilling_np=is_prefilling_np,
|
||||
max_seq_len_np=max_seq_len_np,
|
||||
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
|
||||
positions=self.input_buffers.positions[:num_tokens_after_padding],
|
||||
logits_indices=logits_indices,
|
||||
@@ -1005,12 +1048,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
return sampler_output, num_sampled, num_rejected
|
||||
|
||||
def postprocess(
|
||||
def postprocess_sampled(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
idx_mapping: torch.Tensor, # May include -1 for masked entries
|
||||
sampled_tokens: torch.Tensor,
|
||||
num_sampled: torch.Tensor,
|
||||
num_rejected: torch.Tensor,
|
||||
query_start_loc: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
# Update the number of computed tokens.
|
||||
if self.is_last_pp_rank:
|
||||
@@ -1019,19 +1063,19 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
else:
|
||||
output_bin_counts = None
|
||||
post_update(
|
||||
input_batch.idx_mapping,
|
||||
idx_mapping,
|
||||
self.req_states.num_computed_tokens.gpu,
|
||||
self.req_states.last_sampled_tokens,
|
||||
output_bin_counts,
|
||||
sampled_tokens,
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
input_batch.query_start_loc,
|
||||
query_start_loc,
|
||||
self.req_states.all_token_ids.gpu,
|
||||
self.req_states.total_len.gpu,
|
||||
)
|
||||
|
||||
self.model_state.postprocess_state(input_batch, num_sampled)
|
||||
self.model_state.postprocess_state(idx_mapping, num_sampled)
|
||||
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
@@ -1044,6 +1088,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
) -> ModelRunnerOutput | IntermediateTensors | None:
|
||||
if not dummy_run:
|
||||
# Update the request states.
|
||||
self.update_pp_decode_requests()
|
||||
self.finish_requests(scheduler_output)
|
||||
self.free_states(scheduler_output)
|
||||
self.add_requests(scheduler_output)
|
||||
@@ -1162,9 +1207,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# NOTE(woosuk): We must call get_mm_embeddings even during dummy runs
|
||||
# to obtain inputs_embeds, because the compiled model expects this input.
|
||||
inputs_embeds = self.model_state.get_mm_embeddings(
|
||||
scheduler_output.scheduled_encoder_inputs,
|
||||
input_batch,
|
||||
self.req_states,
|
||||
scheduler_output.scheduled_encoder_inputs, input_batch
|
||||
)
|
||||
|
||||
model_inputs = {
|
||||
@@ -1184,12 +1227,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
assert intermediate_tensors is not None
|
||||
assert self.intermediate_tensors is not None
|
||||
n = input_batch.num_tokens_after_padding
|
||||
model_inputs["intermediate_tensors"] = IntermediateTensors(
|
||||
{
|
||||
k: v[:n].copy_(intermediate_tensors.tensors[k][:n])
|
||||
for k, v in self.intermediate_tensors.tensors.items()
|
||||
}
|
||||
)
|
||||
new_tensors = {
|
||||
k: v[:n]
|
||||
if dummy_run
|
||||
else v[:n].copy_(intermediate_tensors.tensors[k][:n])
|
||||
for k, v in self.intermediate_tensors.tensors.items()
|
||||
}
|
||||
model_inputs["intermediate_tensors"] = IntermediateTensors(new_tensors)
|
||||
del intermediate_tensors
|
||||
|
||||
# Run model.
|
||||
@@ -1281,10 +1325,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Non-last PP rank: hidden_states is None because this rank produced
|
||||
# IntermediateTensors instead of final hidden states. Receive the
|
||||
# sampled tokens broadcast from the last rank and update local state.
|
||||
sampled, num_sampled, num_rejected = pp_receive(
|
||||
input_batch.num_reqs, max_sample_len=self.num_speculative_steps + 1
|
||||
)
|
||||
self.postprocess(input_batch, sampled, num_sampled, num_rejected)
|
||||
assert self.pp_handler is not None
|
||||
all_decode_next = self.pp_handler.receive(input_batch)
|
||||
# Optimistically update num_computed_tokens for entire batch here.
|
||||
# Will be adjusted for rejections if necessary in update_requests.
|
||||
self.postprocess_num_computed_tokens(input_batch)
|
||||
if not all_decode_next:
|
||||
# Might contain non-final prefill chunks, which will be scheduled
|
||||
# in the immediate next step (rather than in pp_size steps).
|
||||
self.model_state.postprocess_state(input_batch.idx_mapping, 0)
|
||||
|
||||
# Post-step KV connector related operations.
|
||||
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
||||
@@ -1295,9 +1344,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
hidden_states, input_batch, grammar_output
|
||||
)
|
||||
|
||||
if self.use_pp:
|
||||
if self.pp_handler is not None:
|
||||
# Broadcast to non-last PP ranks (handles spec decode multi-token).
|
||||
pp_broadcast(sampler_output.sampled_token_ids, num_sampled, num_rejected)
|
||||
self.pp_handler.broadcast(
|
||||
sampler_output.sampled_token_ids,
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
input_batch,
|
||||
)
|
||||
|
||||
assert self.prompt_logprobs_worker is not None
|
||||
prompt_logprobs_dict = self.prompt_logprobs_worker.compute_prompt_logprobs(
|
||||
@@ -1307,8 +1361,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.req_states.all_token_ids.gpu,
|
||||
self.req_states.num_computed_tokens.gpu,
|
||||
self.req_states.prompt_len.np,
|
||||
self.req_states.prefill_len.np,
|
||||
self.req_states.num_computed_prefill_tokens,
|
||||
)
|
||||
|
||||
# Prepare the model runner output.
|
||||
@@ -1334,17 +1386,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Get cached multimodal embeddings for draft forward.
|
||||
# NOTE: This is done here because postprocess updates
|
||||
# num_computed_prefill_tokens.
|
||||
prefill_lens = self.req_states.prefill_len.np[input_batch.idx_mapping_np]
|
||||
computed_prefill_lens = self.req_states.num_computed_prefill_tokens[
|
||||
input_batch.idx_mapping_np
|
||||
]
|
||||
mm_inputs = self.model_state.encoder_runner.gather_mm_embeddings(
|
||||
input_batch.req_ids,
|
||||
input_batch.num_tokens,
|
||||
input_batch.num_scheduled_tokens,
|
||||
input_batch.query_start_loc_np,
|
||||
prefill_lens,
|
||||
computed_prefill_lens + 1, # +1 to consider the skew in eagle
|
||||
input_batch.prefill_len_np,
|
||||
# +1 to consider the skew in eagle
|
||||
input_batch.num_computed_prefill_tokens_np + 1,
|
||||
)
|
||||
|
||||
# Postprocess results and update request states.
|
||||
@@ -1352,8 +1401,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# ensuring that `copy_event` is recorded before calling postprocess.
|
||||
# This sequencing may slightly reduce latency as async D2H copy does not
|
||||
# need to wait for the postprocess to finish.
|
||||
self.postprocess(
|
||||
input_batch, sampler_output.sampled_token_ids, num_sampled, num_rejected
|
||||
self.postprocess_sampled(
|
||||
input_batch.idx_mapping,
|
||||
sampler_output.sampled_token_ids,
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
input_batch.query_start_loc,
|
||||
)
|
||||
|
||||
if self.speculator is not None:
|
||||
@@ -1410,7 +1463,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
||||
|
||||
if not self.is_last_pp_rank:
|
||||
self.postprocess_pool(input_batch)
|
||||
self.postprocess_num_computed_tokens(input_batch)
|
||||
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
||||
|
||||
assert self.pooling_runner is not None
|
||||
@@ -1432,14 +1485,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
copy_stream=self.output_copy_stream,
|
||||
)
|
||||
|
||||
self.postprocess_pool(input_batch)
|
||||
self.postprocess_num_computed_tokens(input_batch)
|
||||
if self.use_async_scheduling:
|
||||
return async_output
|
||||
return async_output.get_output()
|
||||
|
||||
def postprocess_pool(self, input_batch: InputBatch) -> None:
|
||||
def postprocess_num_computed_tokens(self, input_batch: InputBatch) -> None:
|
||||
# Update the number of computed tokens.
|
||||
post_update_pool(
|
||||
post_update_num_computed_tokens(
|
||||
input_batch.idx_mapping,
|
||||
self.req_states.num_computed_tokens.gpu,
|
||||
input_batch.query_start_loc,
|
||||
|
||||
@@ -102,7 +102,6 @@ class DefaultModelState(ModelState):
|
||||
self,
|
||||
scheduled_encoder_inputs: dict[str, list[int]],
|
||||
input_batch: InputBatch,
|
||||
req_states: RequestState,
|
||||
) -> torch.Tensor:
|
||||
mm_hashes, mm_kwargs = self.encoder_runner.prepare_mm_inputs(
|
||||
scheduled_encoder_inputs
|
||||
@@ -118,8 +117,8 @@ class DefaultModelState(ModelState):
|
||||
input_batch.num_tokens,
|
||||
input_batch.num_scheduled_tokens,
|
||||
input_batch.query_start_loc_np,
|
||||
req_states.prefill_len.np[input_batch.idx_mapping_np],
|
||||
req_states.num_computed_prefill_tokens[input_batch.idx_mapping_np],
|
||||
input_batch.prefill_len_np,
|
||||
input_batch.num_computed_prefill_tokens_np,
|
||||
)
|
||||
# Use unpadded input_ids to match is_mm_embed size (num_tokens).
|
||||
# input_batch.input_ids may be padded for CUDA graphs.
|
||||
|
||||
@@ -57,18 +57,13 @@ class ModelState(ABC):
|
||||
return None
|
||||
|
||||
def postprocess_state(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
num_sampled: torch.Tensor,
|
||||
self, idx_mapping: torch.Tensor, num_sampled: torch.Tensor
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
@abstractmethod
|
||||
def get_mm_embeddings(
|
||||
self,
|
||||
scheduled_encoder_inputs: dict[str, list[int]],
|
||||
input_batch: InputBatch,
|
||||
req_states: RequestState,
|
||||
self, scheduled_encoder_inputs: dict[str, list[int]], input_batch: InputBatch
|
||||
) -> torch.Tensor | None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
@@ -112,13 +113,10 @@ class MambaHybridModelState(DefaultModelState):
|
||||
# need the -1 sentinel rather than a raw zero draft count.
|
||||
num_decode_draft_tokens_np = np.full(num_reqs, -1, dtype=np.int32)
|
||||
if input_batch.num_draft_tokens_per_req is not None:
|
||||
spec_decode_mask = (
|
||||
input_batch.num_draft_tokens_per_req > 0
|
||||
) & ~input_batch.is_prefilling_np
|
||||
has_draft_tokens = input_batch.num_draft_tokens_per_req > 0
|
||||
spec_decode_mask = has_draft_tokens & ~input_batch.is_prefilling_np
|
||||
num_decode_draft_tokens_np[: input_batch.num_reqs] = np.where(
|
||||
spec_decode_mask,
|
||||
input_batch.num_draft_tokens_per_req,
|
||||
-1,
|
||||
spec_decode_mask, input_batch.num_draft_tokens_per_req, -1
|
||||
)
|
||||
num_decode_draft_tokens_cpu = torch.from_numpy(num_decode_draft_tokens_np)
|
||||
|
||||
@@ -145,12 +143,33 @@ class MambaHybridModelState(DefaultModelState):
|
||||
)
|
||||
|
||||
def postprocess_state(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
num_sampled: torch.Tensor,
|
||||
self, idx_mapping: torch.Tensor, num_sampled: torch.Tensor | int
|
||||
) -> None:
|
||||
# Chunked prefill does not sample a token, so num_sampled can be 0.
|
||||
# Mamba treats num_accepted_tokens=1 as the neutral non-spec value.
|
||||
self.num_accepted_tokens_gpu[input_batch.idx_mapping] = torch.clamp(
|
||||
num_sampled, min=1
|
||||
)
|
||||
if not isinstance(num_sampled, int):
|
||||
# idx_mapping may contain -1 sentinels (filtered rows) under PP; the
|
||||
# kernel skips them rather than scattering with a host-side gather.
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
if num_reqs:
|
||||
_scatter_num_accepted_kernel[(num_reqs,)](
|
||||
idx_mapping, num_sampled, self.num_accepted_tokens_gpu
|
||||
)
|
||||
return
|
||||
|
||||
# Fill with single value.
|
||||
self.num_accepted_tokens_gpu.index_fill_(0, idx_mapping, max(num_sampled, 1))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _scatter_num_accepted_kernel(
|
||||
idx_mapping_ptr, # [num_reqs] batch_idx -> req_state_idx (-1 to skip)
|
||||
num_sampled_ptr, # [num_reqs]
|
||||
num_accepted_ptr, # [max_num_reqs]
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
req_state_idx = tl.load(idx_mapping_ptr + row)
|
||||
if req_state_idx < 0:
|
||||
return
|
||||
num_sampled = tl.load(num_sampled_ptr + row)
|
||||
tl.store(num_accepted_ptr + req_state_idx, tl.maximum(num_sampled, 1))
|
||||
|
||||
@@ -84,10 +84,7 @@ class WhisperModelState(ModelState):
|
||||
return ("transcription",)
|
||||
|
||||
def get_mm_embeddings(
|
||||
self,
|
||||
scheduled_encoder_inputs: dict[str, list[int]],
|
||||
input_batch: InputBatch,
|
||||
req_states: RequestState,
|
||||
self, scheduled_encoder_inputs: dict[str, list[int]], input_batch: InputBatch
|
||||
) -> None:
|
||||
# Ensure encoder inputs are ordered consistently with input_batch.req_ids.
|
||||
encoder_inputs: dict[str, list[int]] = {}
|
||||
|
||||
+179
-26
@@ -2,40 +2,193 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Pipeline Parallelism utils for V2 Model Runner."""
|
||||
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
|
||||
|
||||
def pp_broadcast(
|
||||
sampled_token_ids: torch.Tensor,
|
||||
num_sampled: torch.Tensor,
|
||||
num_rejected: torch.Tensor,
|
||||
) -> None:
|
||||
pp = get_pp_group()
|
||||
assert pp.is_last_rank
|
||||
@dataclass
|
||||
class PendingRecv:
|
||||
"""Per-step slot data for a deferred postprocess on the main stream."""
|
||||
|
||||
assert sampled_token_ids.dtype == torch.int64
|
||||
torch.distributed.broadcast(
|
||||
sampled_token_ids.contiguous(), src=pp.last_rank, group=pp.device_group
|
||||
)
|
||||
event: torch.cuda.Event
|
||||
|
||||
combined = torch.stack((num_sampled, num_rejected), dim=0)
|
||||
torch.distributed.broadcast(combined, src=pp.last_rank, group=pp.device_group)
|
||||
sampled_tokens: torch.Tensor # [num_reqs, max_sample_len]
|
||||
num_sampled: torch.Tensor # [num_reqs]
|
||||
num_rejected: torch.Tensor # [num_reqs]
|
||||
idx_mapping: torch.Tensor # [num_reqs]
|
||||
idx_mapping_np: np.ndarray # [num_reqs]
|
||||
# Records which rows need a deferred postprocess (bool).
|
||||
need_sampled_mask: np.ndarray # [num_reqs]
|
||||
# Snapshot of slot generation counters at receive time, used to
|
||||
# detect requests aborted since then.
|
||||
gen_at_receive_np: np.ndarray # [num_reqs]
|
||||
|
||||
|
||||
def pp_receive(
|
||||
num_reqs: int, max_sample_len: int = 1
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
pp = get_pp_group()
|
||||
assert not pp.is_last_rank
|
||||
def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None:
|
||||
"""Return a bool array of shape `[input_batch.num_reqs]` marking requests
|
||||
with outputs that might be needed in a subsequent (decode) step.
|
||||
Returns None if no sampled outputs are needed in the requests' next step."""
|
||||
|
||||
sampled_tokens = torch.empty(
|
||||
num_reqs, max_sample_len, dtype=torch.int64, device=pp.device
|
||||
)
|
||||
torch.distributed.broadcast(sampled_tokens, src=pp.last_rank, group=pp.device_group)
|
||||
old_computed = input_batch.num_computed_tokens_np
|
||||
prefill_len = input_batch.prefill_len_np
|
||||
max_seq_len = input_batch.max_seq_len_np
|
||||
assert max_seq_len is not None # always populated under PP
|
||||
# Exclude non-final prefill chunks (they don't produce a sample).
|
||||
produces_sample = old_computed + input_batch.num_scheduled_tokens >= prefill_len
|
||||
# Exclude requests that we know are finished.
|
||||
not_finishing = np.maximum(old_computed, prefill_len) + 1 < max_seq_len
|
||||
need_sampled_mask = produces_sample & not_finishing
|
||||
return need_sampled_mask if need_sampled_mask.any() else None
|
||||
|
||||
combined = torch.empty(2, num_reqs, dtype=torch.int32, device=pp.device)
|
||||
torch.distributed.broadcast(combined, src=pp.last_rank, group=pp.device_group)
|
||||
num_sampled, num_rejected = combined.unbind(dim=0)
|
||||
return sampled_tokens, num_sampled, num_rejected
|
||||
|
||||
class PPHandler:
|
||||
"""Runs the PP sampled-token broadcast/recv on a side stream so the
|
||||
default stream isn't gated by the matching peer call. Step T's recv is
|
||||
consumed at step T+pp_size via `get_prev_sampled_outputs`.
|
||||
|
||||
Uses a dedicated NCCL communicator (sibling of the PP `device_group`)
|
||||
for the broadcast so it does not serialize on the wire with the
|
||||
inter-stage hidden-state p2p send/recv ops.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, max_num_reqs: int, num_speculative_steps: int, device: torch.device
|
||||
):
|
||||
self.is_last_rank = get_pp_group().is_last_rank
|
||||
self.last_rank = get_pp_group().last_rank
|
||||
self.max_sample_len = num_speculative_steps + 1
|
||||
self.device = device
|
||||
self.main_stream = torch.cuda.current_stream(device)
|
||||
self.broadcast_stream = torch.cuda.Stream(device)
|
||||
|
||||
# On non-last ranks, a FIFO with one entry per in-flight step: the entry
|
||||
# pushed by step T's `receive` is consumed pp_size steps later. Pre-seeded
|
||||
# with pp_size None placeholders so the first pp_size consumes are no-ops.
|
||||
# None means no postprocess is pending for that step (broadcast skipped).
|
||||
self.queue: deque[PendingRecv | None] = (
|
||||
deque() if self.is_last_rank else deque([None] * get_pp_group().world_size)
|
||||
)
|
||||
|
||||
# Per req-index generation counter, incremented every time a request
|
||||
# index is freed in RequestStats. Used for invalidating freed req data
|
||||
# between PP decodes.
|
||||
self.req_idx_gen_np = np.zeros(max_num_reqs, dtype=np.int32)
|
||||
|
||||
# Dedicated subgroup for the sampled-token broadcast.
|
||||
self.broadcast_group = get_pp_group().make_sibling_device_group(
|
||||
group_desc="pp_broadcast"
|
||||
)
|
||||
|
||||
def on_req_idx_freed(self, req_idx: int) -> None:
|
||||
self.req_idx_gen_np[req_idx] += 1
|
||||
|
||||
def get_prev_sampled_outputs(self) -> dict[str, torch.Tensor] | None:
|
||||
"""Consume the entry from pp_size steps ago and wait for its recv event,
|
||||
then filter out entries whose request was freed since `receive`.
|
||||
"""
|
||||
if not self.queue:
|
||||
return None
|
||||
slot = self.queue.popleft()
|
||||
# Reserve this step's slot; `receive` overwrites it if applicable.
|
||||
self.queue.append(None)
|
||||
if slot is None:
|
||||
return None
|
||||
|
||||
# Skip requests which did not need sampled output and/or those already
|
||||
# finished. The post_update kernel skips the -1 entries.
|
||||
freed = self.req_idx_gen_np[slot.idx_mapping_np] != slot.gen_at_receive_np
|
||||
exclude_mask = freed | ~slot.need_sampled_mask
|
||||
idx_mapping = slot.idx_mapping
|
||||
if exclude_mask.any():
|
||||
if exclude_mask.all():
|
||||
# No states require update anymore.
|
||||
return None
|
||||
# Filter excluded request indices.
|
||||
idx_mapping_np = np.where(exclude_mask, -1, slot.idx_mapping_np)
|
||||
idx_mapping = async_copy_to_gpu(idx_mapping_np, device=self.device)
|
||||
|
||||
self.main_stream.wait_event(slot.event)
|
||||
return dict(
|
||||
sampled_tokens=slot.sampled_tokens,
|
||||
num_sampled=slot.num_sampled,
|
||||
num_rejected=slot.num_rejected,
|
||||
idx_mapping=idx_mapping,
|
||||
)
|
||||
|
||||
def receive(self, input_batch: InputBatch) -> bool:
|
||||
"""Returns True iff sampled tokens need to be gathered from *all*
|
||||
requests in the batch."""
|
||||
assert not self.is_last_rank
|
||||
need_sampled_mask = compute_need_sampled_mask(input_batch)
|
||||
if need_sampled_mask is None:
|
||||
# Leave this step's reserved slot as None.
|
||||
return False
|
||||
|
||||
# Snapshot the per-slot generation counter so a later free of any of
|
||||
# these RequestStates request indices is detectable at consume time.
|
||||
gen_at_receive_np = self.req_idx_gen_np[input_batch.idx_mapping_np]
|
||||
|
||||
num_reqs = input_batch.num_reqs
|
||||
with torch.cuda.stream(self.broadcast_stream):
|
||||
self.broadcast_stream.wait_stream(self.main_stream)
|
||||
sampled_tokens = torch.empty(
|
||||
num_reqs, self.max_sample_len, dtype=torch.int64, device=self.device
|
||||
)
|
||||
combined = torch.empty(2, num_reqs, dtype=torch.int32, device=self.device)
|
||||
torch.distributed.broadcast(
|
||||
sampled_tokens, src=self.last_rank, group=self.broadcast_group
|
||||
)
|
||||
torch.distributed.broadcast(
|
||||
combined, src=self.last_rank, group=self.broadcast_group
|
||||
)
|
||||
event = self.broadcast_stream.record_event()
|
||||
num_sampled, num_rejected = combined.unbind(dim=0)
|
||||
# Must record_stream since these were allocated on broadcast stream but
|
||||
# later used on the main stream.
|
||||
sampled_tokens.record_stream(self.main_stream)
|
||||
combined.record_stream(self.main_stream)
|
||||
self.queue[-1] = PendingRecv(
|
||||
event,
|
||||
sampled_tokens,
|
||||
num_sampled,
|
||||
num_rejected,
|
||||
input_batch.idx_mapping,
|
||||
input_batch.idx_mapping_np,
|
||||
need_sampled_mask,
|
||||
gen_at_receive_np,
|
||||
)
|
||||
return bool(need_sampled_mask.all())
|
||||
|
||||
def broadcast(
|
||||
self,
|
||||
sampled_token_ids: torch.Tensor,
|
||||
num_sampled: torch.Tensor,
|
||||
num_rejected: torch.Tensor,
|
||||
input_batch: InputBatch,
|
||||
) -> None:
|
||||
assert self.is_last_rank
|
||||
if compute_need_sampled_mask(input_batch) is None:
|
||||
# No request needs sampled outputs for a subsequent decode step.
|
||||
return
|
||||
|
||||
assert sampled_token_ids.dtype == torch.int64
|
||||
with torch.cuda.stream(self.broadcast_stream):
|
||||
self.broadcast_stream.wait_stream(self.main_stream)
|
||||
torch.distributed.broadcast(
|
||||
sampled_token_ids.contiguous(),
|
||||
src=self.last_rank,
|
||||
group=self.broadcast_group,
|
||||
)
|
||||
combined = torch.stack((num_sampled, num_rejected), dim=0)
|
||||
torch.distributed.broadcast(
|
||||
combined, src=self.last_rank, group=self.broadcast_group
|
||||
)
|
||||
for tensor in (sampled_token_ids, num_sampled, num_rejected):
|
||||
tensor.record_stream(self.broadcast_stream)
|
||||
|
||||
@@ -42,10 +42,6 @@ class PromptLogprobsWorker:
|
||||
num_computed_tokens: torch.Tensor,
|
||||
# [max_num_reqs]
|
||||
prompt_lens: np.ndarray,
|
||||
# [max_num_reqs]
|
||||
prefill_lens: np.ndarray,
|
||||
# [max_num_reqs]
|
||||
num_computed_prefill_tokens: np.ndarray,
|
||||
) -> dict[str, LogprobsTensors]:
|
||||
idx_mapping_np = input_batch.idx_mapping_np
|
||||
needs_prompt_logprobs = self.uses_prompt_logprobs[idx_mapping_np]
|
||||
@@ -55,11 +51,11 @@ class PromptLogprobsWorker:
|
||||
|
||||
num_prompt_logprobs = self.num_prompt_logprobs[idx_mapping_np]
|
||||
prompt_lens = prompt_lens[idx_mapping_np]
|
||||
computed_prefill = num_computed_prefill_tokens[idx_mapping_np]
|
||||
computed_prefill = input_batch.num_computed_prefill_tokens_np
|
||||
includes_prompt = computed_prefill < prompt_lens
|
||||
# NOTE(woosuk): If the request was resumed after preemption, its prompt
|
||||
# logprobs must have been computed before preemption. Skip.
|
||||
resumed_after_prompt = prompt_lens < prefill_lens[idx_mapping_np]
|
||||
resumed_after_prompt = prompt_lens < input_batch.prefill_len_np
|
||||
needs_prompt_logprobs &= includes_prompt & ~resumed_after_prompt
|
||||
if not np.any(needs_prompt_logprobs):
|
||||
return {}
|
||||
|
||||
@@ -65,6 +65,9 @@ class RequestState:
|
||||
self.max_num_reqs, 1, dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
# Max total seq length (prompt_len + max_tokens).
|
||||
self.max_seq_len = np.zeros(self.max_num_reqs, dtype=np.int32)
|
||||
|
||||
# Draft tokens.
|
||||
self.draft_tokens = torch.zeros(
|
||||
self.max_num_reqs,
|
||||
@@ -87,12 +90,14 @@ class RequestState:
|
||||
prompt_len: int,
|
||||
all_token_ids: list[int],
|
||||
num_computed_tokens: int,
|
||||
max_tokens: int,
|
||||
) -> None:
|
||||
assert len(self.free_indices) > 0, "No free indices"
|
||||
req_idx = self.free_indices.pop()
|
||||
self.req_id_to_index[req_id] = req_idx
|
||||
self.index_to_req_id[req_idx] = req_id
|
||||
|
||||
self.max_seq_len[req_idx] = prompt_len + max_tokens
|
||||
self.prompt_len.np[req_idx] = prompt_len
|
||||
prefill_len = len(all_token_ids)
|
||||
assert prefill_len >= prompt_len, (
|
||||
@@ -124,17 +129,11 @@ class RequestState:
|
||||
self.all_token_ids.apply_write()
|
||||
self.num_computed_tokens.apply_write()
|
||||
|
||||
def remove_request(self, req_id: str) -> bool:
|
||||
def remove_request(self, req_id: str) -> int | None:
|
||||
"""Return the freed slot index, or None if the request was not found."""
|
||||
req_idx = self.req_id_to_index.pop(req_id, None)
|
||||
if req_idx is None:
|
||||
# Request not found.
|
||||
return False
|
||||
return None
|
||||
self.index_to_req_id.pop(req_idx, None)
|
||||
self.free_indices.append(req_idx)
|
||||
return True
|
||||
|
||||
def is_prefilling(self, idx_mapping_np: np.ndarray) -> np.ndarray:
|
||||
return (
|
||||
self.num_computed_prefill_tokens[idx_mapping_np]
|
||||
< self.prefill_len.np[idx_mapping_np]
|
||||
)
|
||||
return req_idx
|
||||
|
||||
Reference in New Issue
Block a user