[ModelRunnerV2] Avoid pipeline parallel bubbles (#42187)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-06-02 14:02:01 -07:00
committed by GitHub
parent 557781131a
commit e15f20258b
19 changed files with 615 additions and 144 deletions
@@ -94,11 +94,13 @@ steps:
- vllm/v1/worker/gpu_worker.py
- tests/distributed/test_pipeline_parallel.py
- tests/distributed/test_pp_cudagraph.py
- tests/v1/distributed/test_pp_dp_v2.py
commands:
- set -x
- export VLLM_USE_V2_MODEL_RUNNER=1
- pytest -v -s distributed/test_pipeline_parallel.py -k "not ray and not Jamba"
- pytest -v -s distributed/test_pp_cudagraph.py -k "not ray"
- pytest -v -s v1/distributed/test_pp_dp_v2.py
- label: Model Runner V2 Spec Decode
device: h200_35gb
+178
View File
@@ -0,0 +1,178 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""V2 ModelRunner + pipeline parallel + data parallel integration tests.
Covers the interaction between the V2 model runner's PP sampled-token
broadcast and the DP per-step all-reduce across a few concurrency
regimes. Requires 4 GPUs (DP=2, PP=2, TP=1) on CUDA.
"""
import asyncio
import contextlib
import os
from contextlib import ExitStack
import pytest
from vllm import SamplingParams
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.platforms import current_platform
from vllm.sampling_params import RequestOutputKind
from vllm.v1.engine.async_llm import AsyncLLM
PP_DP_MODEL = "ibm-research/PowerMoE-3b" # smallest cached MoE that supports PP
PROMPT = "This is a test of data parallel and pipeline parallel together"
def _gpu_skip_reason() -> str | None:
if not current_platform.is_cuda():
return "requires CUDA"
n = current_platform.device_count()
if n < 4:
return f"requires 4 GPUs, got {n}"
return None
_GPU_SKIP = _gpu_skip_reason()
pytestmark = [
pytest.mark.skipif(
os.environ.get("VLLM_USE_V2_MODEL_RUNNER", "0") != "1",
reason="VLLM_USE_V2_MODEL_RUNNER=1 required",
),
pytest.mark.skipif(_GPU_SKIP is not None, reason=_GPU_SKIP or ""),
]
def _engine_args(async_scheduling: bool) -> AsyncEngineArgs:
return AsyncEngineArgs(
model=PP_DP_MODEL,
pipeline_parallel_size=2,
data_parallel_size=2,
data_parallel_backend="mp",
tensor_parallel_size=1,
max_model_len=4096,
max_num_batched_tokens=2048,
max_num_seqs=256,
async_scheduling=async_scheduling,
enable_prefix_caching=False,
enforce_eager=False,
enable_expert_parallel=False,
)
async def _generate(engine: AsyncLLM, prompt: str, max_tokens: int) -> int:
"""Run one streaming completion and return the number of tokens it yielded."""
sampling_params = SamplingParams(
max_tokens=max_tokens,
ignore_eos=True,
output_kind=RequestOutputKind.DELTA,
temperature=0.0,
)
request_id = f"req-{id(prompt):x}-{max_tokens}"
total = 0
async for out in engine.generate(
request_id=request_id, prompt=prompt, sampling_params=sampling_params
):
total += len(out.outputs[0].token_ids)
return total
@pytest.mark.asyncio
@pytest.mark.parametrize("async_scheduling", [True, False])
async def test_pp_dp_v2_low_concurrency(async_scheduling: bool):
"""A single in-flight request at a time, repeated, to exercise the
PP slot ring under empty batches between decodes."""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling))
after.callback(engine.shutdown)
for _ in range(4):
n = await _generate(engine, PROMPT, max_tokens=16)
assert n == 16
@pytest.mark.asyncio
@pytest.mark.parametrize("async_scheduling", [True, False])
async def test_pp_dp_v2_mid_concurrency(async_scheduling: bool):
"""64 concurrent requests, staggered, to exercise the steady-state
DP all-reduce + PP slot-ring path."""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling))
after.callback(engine.shutdown)
async def _one(i: int) -> int:
await asyncio.sleep(0.01 * i) # stagger so DP load-balances
return await _generate(engine, f"{PROMPT} {i}", max_tokens=64)
results = await asyncio.gather(*[_one(i) for i in range(64)])
assert all(n == 64 for n in results), results
@pytest.mark.asyncio
async def test_pp_dp_v2_abort_mid_decode():
"""Cancel half the in-flight requests mid-stream and confirm the
engine survives the abort storm."""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling=True))
after.callback(engine.shutdown)
async def _maybe_cancel(i: int):
sampling_params = SamplingParams(
max_tokens=64,
ignore_eos=True,
output_kind=RequestOutputKind.DELTA,
temperature=0.0,
)
request_id = f"abort-req-{i}"
count = 0
cancel_at = 4 if i % 2 == 0 else 64
async for out in engine.generate(
request_id=request_id,
prompt=f"{PROMPT} {i}",
sampling_params=sampling_params,
):
count += len(out.outputs[0].token_ids)
if count >= cancel_at:
break
return count, i
results = await asyncio.gather(*[_maybe_cancel(i) for i in range(32)])
for count, i in results:
if i % 2 == 0:
assert count >= 4
else:
assert count == 64
# Engine must still serve after the abort storm.
final = await _generate(engine, "post-abort warmup", max_tokens=8)
assert final == 8
@pytest.mark.asyncio
async def test_pp_dp_v2_pause_resume():
"""Pause an engine with a request in flight, then resume and confirm
new requests still work."""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(_engine_args(async_scheduling=True))
after.callback(engine.shutdown)
# Start a long-running generation, let some decoding happen, then
# pause (abort mode) and confirm the in-flight task terminates.
inflight = asyncio.create_task(_generate(engine, PROMPT, max_tokens=128))
await asyncio.sleep(0.5)
assert not await engine.is_paused()
await engine.pause_generation(mode="abort")
assert await engine.is_paused()
with contextlib.suppress(Exception):
await inflight
await engine.resume_generation()
assert not await engine.is_paused()
n = await _generate(engine, PROMPT, max_tokens=8)
assert n == 8
@@ -167,7 +167,9 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
events = []
runner = _make_runner(is_last_pp_rank=False, num_speculative_steps=0)
runner.execute_model_state = SimpleNamespace(
input_batch=SimpleNamespace(num_reqs=2),
input_batch=SimpleNamespace(
num_reqs=2, idx_mapping=torch.zeros(2, dtype=torch.int32)
),
attn_metadata=None,
slot_mappings_by_layer=None,
hidden_states=None,
@@ -175,18 +177,19 @@ def test_v2_sample_tokens_runs_eplb_on_non_last_pp_rank(monkeypatch):
finished_req_ids=set(),
num_tokens_across_dp=None,
)
runner.postprocess = lambda *args, **kwargs: events.append("postprocess")
runner.eplb.step = lambda *args, **kwargs: events.append("eplb")
monkeypatch.setattr(
mrv2,
"pp_receive",
lambda *args, **kwargs: (
torch.zeros((2, 1), dtype=torch.long),
torch.ones(2, dtype=torch.int32),
torch.zeros(2, dtype=torch.int32),
),
runner.req_states = SimpleNamespace()
def fake_receive(*args, **kwargs):
events.append("receive")
# all_decode_next=True, so model_state.postprocess_state is skipped.
return True
runner.pp_handler = SimpleNamespace(receive=fake_receive)
runner.postprocess_num_computed_tokens = lambda *args, **kwargs: events.append(
"postprocess_num_computed_tokens"
)
runner.eplb.step = lambda *args, **kwargs: events.append("eplb")
output = mrv2.GPUModelRunner.sample_tokens(runner, None)
assert output in (EMPTY_MODEL_RUNNER_OUTPUT, None)
assert events == ["postprocess", "eplb"]
assert events == ["receive", "postprocess_num_computed_tokens", "eplb"]
+7 -3
View File
@@ -498,9 +498,13 @@ class VllmConfig:
# PP requires PP-size concurrent batches to fill the pipeline.
# Async scheduling requires 2 concurrent batches to overlap.
pp_size = self.parallel_config.pipeline_parallel_size
if pp_size > 1:
return pp_size
return 2 if self.scheduler_config.async_scheduling else 1
if self.scheduler_config.async_scheduling:
if self.use_v2_model_runner:
return pp_size + 1
# V1 Model Runner does not fully support async scheduling with PP.
if pp_size <= 1:
return 2
return pp_size
@property
def num_speculative_tokens(self) -> int:
+19
View File
@@ -359,6 +359,9 @@ class GroupCoordinator:
assert self_cpu_group is not None
assert self_device_group is not None
self.group_ranks = group_ranks
self.torch_distributed_backend = torch_distributed_backend
self.cpu_group = self_cpu_group
self.device_group = self_device_group
@@ -406,6 +409,22 @@ class GroupCoordinator:
and getattr(self.device_communicator, "supports_tensor_dict", False)
)
def make_sibling_device_group(self, group_desc: str | None = None) -> ProcessGroup:
"""Create a new device-side ProcessGroup with the same per-rank membership
as this coordinator's `device_group`, but backed by a distinct communicator.
This is a collective call: every world rank must invoke it. Used where we
want to issue ops that can run concurrently with ops on `device_group`.
"""
sibling: ProcessGroup | None = None
for ranks in self.group_ranks:
pg = torch.distributed.new_group(
ranks, backend=self.torch_distributed_backend, group_desc=group_desc
)
if self.rank in ranks:
sibling = pg
assert sibling is not None
return sibling
def create_mq_broadcaster(
self, writer_rank=0, external_writer_handle=None, blocking=True
):
+6
View File
@@ -14,6 +14,7 @@ class AsyncScheduler(Scheduler):
super().__init__(*args, **kwargs)
# reusable read-only placeholder list for speculative decoding.
self._spec_token_placeholders: list[int] = [-1] * self.num_spec_tokens
self.pp_size = self.parallel_config.pipeline_parallel_size
def _update_after_schedule(self, scheduler_output: SchedulerOutput) -> None:
super()._update_after_schedule(scheduler_output)
@@ -34,6 +35,11 @@ class AsyncScheduler(Scheduler):
# We will update the actual spec token ids in the worker process.
request.spec_token_ids = self._spec_token_placeholders
if self.use_v2_model_runner:
# Set the next step index in which this request is eligible to be
# scheduled for decode (for PP microbatching).
request.next_decode_eligible_step = self.current_step + self.pp_size
def _update_request_with_output(
self, request: Request, new_token_ids: list[int]
) -> tuple[list[int], bool]:
+10
View File
@@ -248,6 +248,9 @@ class Scheduler(SchedulerInterface):
self.use_pp = self.parallel_config.pipeline_parallel_size > 1
self.use_v2_model_runner = vllm_config.use_v2_model_runner
# Scheduler iteration counter. Drives the V2+PP+async decode-throttle
# cadence (`next_decode_eligible_step`).
self.current_step = 0
self.scheduler_reserve_full_isl = (
self.scheduler_config.scheduler_reserve_full_isl
)
@@ -333,6 +336,7 @@ class Scheduler(SchedulerInterface):
return num_new_tokens
def schedule(self) -> SchedulerOutput:
self.current_step += 1
# NOTE(woosuk) on the scheduling algorithm:
# There's no "decoding phase" nor "prefill phase" in the scheduler.
# Each request just has the num_computed_tokens and
@@ -388,6 +392,12 @@ class Scheduler(SchedulerInterface):
req_index += 1
continue
if self.current_step < request.next_decode_eligible_step:
# V2+PP+async: enforce `pp_size` steps between same-req decodes
# to match worker-side sampled-tokens broadcast slot ring cadence.
req_index += 1
continue
num_new_tokens = (
request.num_tokens_with_spec
+ request.num_output_placeholders
+3 -5
View File
@@ -534,14 +534,12 @@ class EngineCore:
if not deferred_scheduler_output:
# Add this step's future to the queue.
batch_queue.appendleft((future, scheduler_output, exec_future))
if (
model_executed
and len(batch_queue) < self.batch_queue_size
and not batch_queue[-1][0].done()
if len(batch_queue) < self.batch_queue_size and (
model_executed or self.scheduler.has_requests()
):
# Don't block on next worker response unless the queue is full
# or there are no more requests to schedule.
return None, True
return None, model_executed
elif not batch_queue:
# Queue is empty. We should not reach here since this method should
+4
View File
@@ -141,6 +141,10 @@ class Request:
self.num_output_placeholders = 0
self.async_tokens_to_discard = 0
# V2+PP+async: Enforces `pp_size` cadence between same-request decode steps
# so the worker's broadcast slot ring stays consistent.
self.next_decode_eligible_step = 0
self.spec_token_ids: list[int] = []
self.num_computed_tokens = 0
self.cache_salt: str | None = cache_salt
+19 -3
View File
@@ -13,6 +13,15 @@ from vllm.utils.torch_utils import (
get_accelerator_view_from_cpu_tensor,
)
# Default round-robin depth for the UVA buffer pools. Must be >= the number of
# concurrent in-flight steps (engine batch_queue_size).
_DEFAULT_MAX_CONCURRENCY = 2
def set_default_max_concurrency(n: int) -> None:
global _DEFAULT_MAX_CONCURRENCY
_DEFAULT_MAX_CONCURRENCY = max(2, n)
def async_copy_to_gpu(
x: torch.Tensor | np.ndarray,
@@ -47,8 +56,10 @@ class UvaBufferPool:
self,
size: int | Sequence[int],
dtype: torch.dtype,
max_concurrency: int = 2,
max_concurrency: int | None = None,
):
if max_concurrency is None:
max_concurrency = _DEFAULT_MAX_CONCURRENCY
self.size = size
self.dtype = dtype
self.max_concurrency = max_concurrency
@@ -80,7 +91,10 @@ class UvaBufferPool:
class UvaBackedTensor:
def __init__(
self, size: int | Sequence[int], dtype: torch.dtype, max_concurrency: int = 2
self,
size: int | Sequence[int],
dtype: torch.dtype,
max_concurrency: int | None = None,
):
self.dtype = dtype
@@ -104,9 +118,11 @@ class StagedWriteTensor:
size: int | Sequence[int],
dtype: torch.dtype,
device: torch.device,
max_concurrency: int = 2,
max_concurrency: int | None = None,
uva_instead_of_gpu: bool = False,
):
if max_concurrency is None:
max_concurrency = _DEFAULT_MAX_CONCURRENCY
supported_dtypes = [torch.int32, torch.int64, torch.float32]
if dtype not in supported_dtypes:
raise ValueError(
+32 -12
View File
@@ -67,9 +67,18 @@ class InputBatch:
seq_lens_cpu_upper_bound: torch.Tensor
# [num_reqs]
dcp_local_seq_lens: torch.Tensor | None
# [num_reqs] CPU bool array.
# [num_reqs]
num_computed_tokens_np: np.ndarray
# [num_reqs]
prefill_len_np: np.ndarray
# [num_reqs]
num_computed_prefill_tokens_np: np.ndarray
# [num_reqs] CPU bool array == (num_computed_prefill_tokens_np < prefill_len_np).
is_prefilling_np: np.ndarray
# [num_reqs] only populated when pipeline parallelism is enabled.
max_seq_len_np: np.ndarray | None
# [num_tokens_after_padding]
input_ids: torch.Tensor
# [num_tokens_after_padding]
@@ -148,7 +157,11 @@ class InputBatch:
seq_lens=seq_lens,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
dcp_local_seq_lens=None,
num_computed_tokens_np=np.zeros(num_reqs, dtype=np.int32),
prefill_len_np=np.zeros(num_reqs, dtype=np.int32),
num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32),
is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_),
max_seq_len_np=None,
input_ids=input_ids,
positions=positions,
logits_indices=logits_indices,
@@ -438,6 +451,9 @@ def _post_update_kernel(
):
req_id = tl.program_id(0)
req_state_idx = tl.load(idx_mapping_ptr + req_id)
if req_state_idx < 0:
# Filter rows with negative index entries.
return
total_len = tl.load(total_len_ptr + req_state_idx)
num_sampled = tl.load(num_sampled_ptr + req_id)
@@ -464,18 +480,22 @@ def _post_update_kernel(
count = tl.load(token_ptr)
tl.store(token_ptr, count + 1)
query_start = tl.load(query_start_loc_ptr + req_id)
query_end = tl.load(query_start_loc_ptr + req_id + 1)
query_len = query_end - query_start
if query_start_loc_ptr is None:
query_len = 0
else:
query_start = tl.load(query_start_loc_ptr + req_id)
query_end = tl.load(query_start_loc_ptr + req_id + 1)
query_len = query_end - query_start
num_rejected = tl.load(num_rejected_ptr + req_id)
num_computed = tl.load(num_computed_tokens_ptr + req_state_idx)
num_computed += query_len - num_rejected
tl.store(num_computed_tokens_ptr + req_state_idx, num_computed)
computed_delta = query_len - num_rejected
if computed_delta != 0:
num_computed = tl.load(num_computed_tokens_ptr + req_state_idx)
tl.store(num_computed_tokens_ptr + req_state_idx, num_computed + computed_delta)
def post_update(
# [num_reqs]
# [num_reqs] batch_idx -> req_state_idx; negative index means skip.
idx_mapping: torch.Tensor,
# [max_num_reqs]
num_computed_tokens: torch.Tensor,
@@ -490,7 +510,7 @@ def post_update(
# [num_reqs]
num_rejected: torch.Tensor,
# [num_reqs + 1]
query_start_loc: torch.Tensor,
query_start_loc: torch.Tensor | None,
# [max_num_reqs, max_model_len]
all_token_ids: torch.Tensor,
# [max_num_reqs]
@@ -516,7 +536,7 @@ def post_update(
@triton.jit
def _post_update_pool_kernel(
def _post_update_num_computed_tokens_kernel(
idx_mapping_ptr,
num_computed_tokens_ptr,
query_start_loc_ptr,
@@ -531,7 +551,7 @@ def _post_update_pool_kernel(
tl.store(num_computed_tokens_ptr + req_state_idx, num_computed + query_len)
def post_update_pool(
def post_update_num_computed_tokens(
# [num_reqs]
idx_mapping: torch.Tensor,
# [max_num_reqs]
@@ -540,7 +560,7 @@ def post_update_pool(
query_start_loc: torch.Tensor,
) -> None:
num_reqs = idx_mapping.shape[0]
_post_update_pool_kernel[(num_reqs,)](
_post_update_num_computed_tokens_kernel[(num_reqs,)](
idx_mapping,
num_computed_tokens,
query_start_loc,
+94 -41
View File
@@ -61,7 +61,10 @@ from vllm.v1.worker.gpu.attn_utils import (
init_kv_cache,
)
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
from vllm.v1.worker.gpu.buffer_utils import (
async_copy_to_gpu,
set_default_max_concurrency,
)
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
from vllm.v1.worker.gpu.cudagraph_utils import (
BatchExecutionDescriptor,
@@ -77,7 +80,7 @@ from vllm.v1.worker.gpu.input_batch import (
expand_idx_mapping,
get_num_sampled_and_rejected,
post_update,
post_update_pool,
post_update_num_computed_tokens,
prepare_pos_seq_lens,
prepare_prefill_inputs,
)
@@ -90,7 +93,7 @@ from vllm.v1.worker.gpu.lora_utils import LoraState
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.model_states import init_model_state
from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner
from vllm.v1.worker.gpu.pp_utils import pp_broadcast, pp_receive
from vllm.v1.worker.gpu.pp_utils import PPHandler
from vllm.v1.worker.gpu.sample.output import SamplerOutput
from vllm.v1.worker.gpu.sample.prompt_logprob import PromptLogprobsWorker
from vllm.v1.worker.gpu.sample.sampler import Sampler
@@ -149,6 +152,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.is_first_pp_rank = get_pp_group().is_first_rank
self.is_last_pp_rank = get_pp_group().is_last_rank
# Size the UVA buffer pools to the max number of concurrent in-flight
# steps. Must run before any pooled buffer is constructed
set_default_max_concurrency(vllm_config.max_concurrent_batches)
# PP broadcast/recv helper. Runs the collective on a side stream.
self.pp_handler: PPHandler | None = None
# Persistent buffer for intermediate tensors (non-first PP ranks).
self.intermediate_tensors: IntermediateTensors | None = None
@@ -210,6 +220,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
device=self.device,
)
if self.use_pp:
self.pp_handler = PPHandler(
max_num_reqs=self.max_num_reqs,
num_speculative_steps=self.num_speculative_steps,
device=self.device,
)
self.sampler: Sampler | None = None
self.rejection_sampler: RejectionSampler | None = None
self.prompt_logprobs_worker: PromptLogprobsWorker | None = None
@@ -688,8 +705,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
return cuda_graph_size
def _remove_request(self, req_id: str) -> bool:
if not self.req_states.remove_request(req_id):
req_idx = self.req_states.remove_request(req_id)
if req_idx is None:
return False
if self.pp_handler is not None:
self.pp_handler.on_req_idx_freed(req_idx)
if self.encoder_cache is not None:
self.encoder_cache.remove_request(req_id)
if self.prompt_logprobs_worker is not None:
@@ -710,6 +730,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
for mm_hash in scheduler_output.free_encoder_mm_hashes:
self.encoder_cache.free_encoder_cache(mm_hash)
def update_pp_decode_requests(self):
# For non-last PP ranks, update decode requests with sampler output from
# the prior step in which they were scheduled (pp_size steps ago).
if self.pp_handler is not None:
outputs = self.pp_handler.get_prev_sampled_outputs()
if outputs is not None:
self.postprocess_sampled(**outputs)
def add_requests(self, scheduler_output: SchedulerOutput) -> None:
for new_req_data in scheduler_output.scheduled_new_reqs:
assert new_req_data.prompt_token_ids is not None
@@ -722,11 +750,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self._remove_request(req_id)
prompt_len = len(new_req_data.prompt_token_ids)
sampling_params = new_req_data.sampling_params
self.req_states.add_request(
req_id=req_id,
prompt_len=prompt_len,
all_token_ids=new_req_data.prefill_token_ids,
num_computed_tokens=new_req_data.num_computed_tokens,
max_tokens=sampling_params.max_tokens if sampling_params else 1, # type: ignore[arg-type]
)
req_index = self.req_states.req_id_to_index[req_id]
@@ -769,7 +799,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
req_index, req_new_block_ids, overwrite=False
)
# Update num_computed_prefill_tokens.
# Update CPU num_computed_prefill_tokens.
np.minimum(
self.req_states.num_computed_tokens_np,
self.req_states.prefill_len.np,
@@ -848,7 +878,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc)
query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1]
query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1]
is_prefilling_np = self.req_states.is_prefilling(idx_mapping_np)
prefill_len_np = self.req_states.prefill_len.np[idx_mapping_np]
computed_prefill_tokens_np = self.req_states.num_computed_prefill_tokens
num_computed_prefill_tokens_np = computed_prefill_tokens_np[idx_mapping_np]
is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np
# Get prefill tokens if any.
if np.any(is_prefilling_np):
@@ -900,13 +933,19 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
# CPU upper bound on seq_lens; padded entries left at zero.
num_computed_tokens_np = self.req_states.num_computed_tokens_np[idx_mapping_np]
seq_lens_cpu_upper_bound_np = np.zeros(num_reqs_padded, dtype=np.int32)
np.add(
self.req_states.num_computed_tokens_np[idx_mapping_np],
num_computed_tokens_np,
num_scheduled_tokens,
out=seq_lens_cpu_upper_bound_np[:num_reqs],
)
seq_lens_cpu_upper_bound = torch.from_numpy(seq_lens_cpu_upper_bound_np)
max_seq_len_np = None
if self.use_pp:
# max_seq_len is only consumed by the PP `compute_need_sampled_mask`
max_seq_len_np = self.req_states.max_seq_len[idx_mapping_np]
return InputBatch(
req_ids=req_ids,
num_reqs=num_reqs,
@@ -925,7 +964,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
seq_lens=seq_lens,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
dcp_local_seq_lens=dcp_local_seq_lens,
num_computed_tokens_np=num_computed_tokens_np,
prefill_len_np=prefill_len_np,
num_computed_prefill_tokens_np=num_computed_prefill_tokens_np,
is_prefilling_np=is_prefilling_np,
max_seq_len_np=max_seq_len_np,
input_ids=self.input_buffers.input_ids[:num_tokens_after_padding],
positions=self.input_buffers.positions[:num_tokens_after_padding],
logits_indices=logits_indices,
@@ -1005,12 +1048,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
return sampler_output, num_sampled, num_rejected
def postprocess(
def postprocess_sampled(
self,
input_batch: InputBatch,
idx_mapping: torch.Tensor, # May include -1 for masked entries
sampled_tokens: torch.Tensor,
num_sampled: torch.Tensor,
num_rejected: torch.Tensor,
query_start_loc: torch.Tensor | None = None,
) -> None:
# Update the number of computed tokens.
if self.is_last_pp_rank:
@@ -1019,19 +1063,19 @@ class GPUModelRunner(LoRAModelRunnerMixin):
else:
output_bin_counts = None
post_update(
input_batch.idx_mapping,
idx_mapping,
self.req_states.num_computed_tokens.gpu,
self.req_states.last_sampled_tokens,
output_bin_counts,
sampled_tokens,
num_sampled,
num_rejected,
input_batch.query_start_loc,
query_start_loc,
self.req_states.all_token_ids.gpu,
self.req_states.total_len.gpu,
)
self.model_state.postprocess_state(input_batch, num_sampled)
self.model_state.postprocess_state(idx_mapping, num_sampled)
@torch.inference_mode()
def execute_model(
@@ -1044,6 +1088,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
) -> ModelRunnerOutput | IntermediateTensors | None:
if not dummy_run:
# Update the request states.
self.update_pp_decode_requests()
self.finish_requests(scheduler_output)
self.free_states(scheduler_output)
self.add_requests(scheduler_output)
@@ -1162,9 +1207,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# NOTE(woosuk): We must call get_mm_embeddings even during dummy runs
# to obtain inputs_embeds, because the compiled model expects this input.
inputs_embeds = self.model_state.get_mm_embeddings(
scheduler_output.scheduled_encoder_inputs,
input_batch,
self.req_states,
scheduler_output.scheduled_encoder_inputs, input_batch
)
model_inputs = {
@@ -1184,12 +1227,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
assert intermediate_tensors is not None
assert self.intermediate_tensors is not None
n = input_batch.num_tokens_after_padding
model_inputs["intermediate_tensors"] = IntermediateTensors(
{
k: v[:n].copy_(intermediate_tensors.tensors[k][:n])
for k, v in self.intermediate_tensors.tensors.items()
}
)
new_tensors = {
k: v[:n]
if dummy_run
else v[:n].copy_(intermediate_tensors.tensors[k][:n])
for k, v in self.intermediate_tensors.tensors.items()
}
model_inputs["intermediate_tensors"] = IntermediateTensors(new_tensors)
del intermediate_tensors
# Run model.
@@ -1281,10 +1325,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Non-last PP rank: hidden_states is None because this rank produced
# IntermediateTensors instead of final hidden states. Receive the
# sampled tokens broadcast from the last rank and update local state.
sampled, num_sampled, num_rejected = pp_receive(
input_batch.num_reqs, max_sample_len=self.num_speculative_steps + 1
)
self.postprocess(input_batch, sampled, num_sampled, num_rejected)
assert self.pp_handler is not None
all_decode_next = self.pp_handler.receive(input_batch)
# Optimistically update num_computed_tokens for entire batch here.
# Will be adjusted for rejections if necessary in update_requests.
self.postprocess_num_computed_tokens(input_batch)
if not all_decode_next:
# Might contain non-final prefill chunks, which will be scheduled
# in the immediate next step (rather than in pp_size steps).
self.model_state.postprocess_state(input_batch.idx_mapping, 0)
# Post-step KV connector related operations.
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
@@ -1295,9 +1344,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
hidden_states, input_batch, grammar_output
)
if self.use_pp:
if self.pp_handler is not None:
# Broadcast to non-last PP ranks (handles spec decode multi-token).
pp_broadcast(sampler_output.sampled_token_ids, num_sampled, num_rejected)
self.pp_handler.broadcast(
sampler_output.sampled_token_ids,
num_sampled,
num_rejected,
input_batch,
)
assert self.prompt_logprobs_worker is not None
prompt_logprobs_dict = self.prompt_logprobs_worker.compute_prompt_logprobs(
@@ -1307,8 +1361,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.req_states.all_token_ids.gpu,
self.req_states.num_computed_tokens.gpu,
self.req_states.prompt_len.np,
self.req_states.prefill_len.np,
self.req_states.num_computed_prefill_tokens,
)
# Prepare the model runner output.
@@ -1334,17 +1386,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Get cached multimodal embeddings for draft forward.
# NOTE: This is done here because postprocess updates
# num_computed_prefill_tokens.
prefill_lens = self.req_states.prefill_len.np[input_batch.idx_mapping_np]
computed_prefill_lens = self.req_states.num_computed_prefill_tokens[
input_batch.idx_mapping_np
]
mm_inputs = self.model_state.encoder_runner.gather_mm_embeddings(
input_batch.req_ids,
input_batch.num_tokens,
input_batch.num_scheduled_tokens,
input_batch.query_start_loc_np,
prefill_lens,
computed_prefill_lens + 1, # +1 to consider the skew in eagle
input_batch.prefill_len_np,
# +1 to consider the skew in eagle
input_batch.num_computed_prefill_tokens_np + 1,
)
# Postprocess results and update request states.
@@ -1352,8 +1401,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# ensuring that `copy_event` is recorded before calling postprocess.
# This sequencing may slightly reduce latency as async D2H copy does not
# need to wait for the postprocess to finish.
self.postprocess(
input_batch, sampler_output.sampled_token_ids, num_sampled, num_rejected
self.postprocess_sampled(
input_batch.idx_mapping,
sampler_output.sampled_token_ids,
num_sampled,
num_rejected,
input_batch.query_start_loc,
)
if self.speculator is not None:
@@ -1410,7 +1463,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
if not self.is_last_pp_rank:
self.postprocess_pool(input_batch)
self.postprocess_num_computed_tokens(input_batch)
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
assert self.pooling_runner is not None
@@ -1432,14 +1485,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
copy_stream=self.output_copy_stream,
)
self.postprocess_pool(input_batch)
self.postprocess_num_computed_tokens(input_batch)
if self.use_async_scheduling:
return async_output
return async_output.get_output()
def postprocess_pool(self, input_batch: InputBatch) -> None:
def postprocess_num_computed_tokens(self, input_batch: InputBatch) -> None:
# Update the number of computed tokens.
post_update_pool(
post_update_num_computed_tokens(
input_batch.idx_mapping,
self.req_states.num_computed_tokens.gpu,
input_batch.query_start_loc,
+2 -3
View File
@@ -102,7 +102,6 @@ class DefaultModelState(ModelState):
self,
scheduled_encoder_inputs: dict[str, list[int]],
input_batch: InputBatch,
req_states: RequestState,
) -> torch.Tensor:
mm_hashes, mm_kwargs = self.encoder_runner.prepare_mm_inputs(
scheduled_encoder_inputs
@@ -118,8 +117,8 @@ class DefaultModelState(ModelState):
input_batch.num_tokens,
input_batch.num_scheduled_tokens,
input_batch.query_start_loc_np,
req_states.prefill_len.np[input_batch.idx_mapping_np],
req_states.num_computed_prefill_tokens[input_batch.idx_mapping_np],
input_batch.prefill_len_np,
input_batch.num_computed_prefill_tokens_np,
)
# Use unpadded input_ids to match is_mm_embed size (num_tokens).
# input_batch.input_ids may be padded for CUDA graphs.
+2 -7
View File
@@ -57,18 +57,13 @@ class ModelState(ABC):
return None
def postprocess_state(
self,
input_batch: InputBatch,
num_sampled: torch.Tensor,
self, idx_mapping: torch.Tensor, num_sampled: torch.Tensor
) -> None:
return None
@abstractmethod
def get_mm_embeddings(
self,
scheduled_encoder_inputs: dict[str, list[int]],
input_batch: InputBatch,
req_states: RequestState,
self, scheduled_encoder_inputs: dict[str, list[int]], input_batch: InputBatch
) -> torch.Tensor | None:
raise NotImplementedError
+31 -12
View File
@@ -9,6 +9,7 @@ import torch.nn as nn
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadataBuilder
from vllm.v1.kv_cache_interface import KVCacheConfig
@@ -112,13 +113,10 @@ class MambaHybridModelState(DefaultModelState):
# need the -1 sentinel rather than a raw zero draft count.
num_decode_draft_tokens_np = np.full(num_reqs, -1, dtype=np.int32)
if input_batch.num_draft_tokens_per_req is not None:
spec_decode_mask = (
input_batch.num_draft_tokens_per_req > 0
) & ~input_batch.is_prefilling_np
has_draft_tokens = input_batch.num_draft_tokens_per_req > 0
spec_decode_mask = has_draft_tokens & ~input_batch.is_prefilling_np
num_decode_draft_tokens_np[: input_batch.num_reqs] = np.where(
spec_decode_mask,
input_batch.num_draft_tokens_per_req,
-1,
spec_decode_mask, input_batch.num_draft_tokens_per_req, -1
)
num_decode_draft_tokens_cpu = torch.from_numpy(num_decode_draft_tokens_np)
@@ -145,12 +143,33 @@ class MambaHybridModelState(DefaultModelState):
)
def postprocess_state(
self,
input_batch: InputBatch,
num_sampled: torch.Tensor,
self, idx_mapping: torch.Tensor, num_sampled: torch.Tensor | int
) -> None:
# Chunked prefill does not sample a token, so num_sampled can be 0.
# Mamba treats num_accepted_tokens=1 as the neutral non-spec value.
self.num_accepted_tokens_gpu[input_batch.idx_mapping] = torch.clamp(
num_sampled, min=1
)
if not isinstance(num_sampled, int):
# idx_mapping may contain -1 sentinels (filtered rows) under PP; the
# kernel skips them rather than scattering with a host-side gather.
num_reqs = idx_mapping.shape[0]
if num_reqs:
_scatter_num_accepted_kernel[(num_reqs,)](
idx_mapping, num_sampled, self.num_accepted_tokens_gpu
)
return
# Fill with single value.
self.num_accepted_tokens_gpu.index_fill_(0, idx_mapping, max(num_sampled, 1))
@triton.jit
def _scatter_num_accepted_kernel(
idx_mapping_ptr, # [num_reqs] batch_idx -> req_state_idx (-1 to skip)
num_sampled_ptr, # [num_reqs]
num_accepted_ptr, # [max_num_reqs]
):
row = tl.program_id(0)
req_state_idx = tl.load(idx_mapping_ptr + row)
if req_state_idx < 0:
return
num_sampled = tl.load(num_sampled_ptr + row)
tl.store(num_accepted_ptr + req_state_idx, tl.maximum(num_sampled, 1))
+1 -4
View File
@@ -84,10 +84,7 @@ class WhisperModelState(ModelState):
return ("transcription",)
def get_mm_embeddings(
self,
scheduled_encoder_inputs: dict[str, list[int]],
input_batch: InputBatch,
req_states: RequestState,
self, scheduled_encoder_inputs: dict[str, list[int]], input_batch: InputBatch
) -> None:
# Ensure encoder inputs are ordered consistently with input_batch.req_ids.
encoder_inputs: dict[str, list[int]] = {}
+179 -26
View File
@@ -2,40 +2,193 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Pipeline Parallelism utils for V2 Model Runner."""
from collections import deque
from dataclasses import dataclass
import numpy as np
import torch
from vllm.distributed.parallel_state import get_pp_group
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
from vllm.v1.worker.gpu.input_batch import InputBatch
def pp_broadcast(
sampled_token_ids: torch.Tensor,
num_sampled: torch.Tensor,
num_rejected: torch.Tensor,
) -> None:
pp = get_pp_group()
assert pp.is_last_rank
@dataclass
class PendingRecv:
"""Per-step slot data for a deferred postprocess on the main stream."""
assert sampled_token_ids.dtype == torch.int64
torch.distributed.broadcast(
sampled_token_ids.contiguous(), src=pp.last_rank, group=pp.device_group
)
event: torch.cuda.Event
combined = torch.stack((num_sampled, num_rejected), dim=0)
torch.distributed.broadcast(combined, src=pp.last_rank, group=pp.device_group)
sampled_tokens: torch.Tensor # [num_reqs, max_sample_len]
num_sampled: torch.Tensor # [num_reqs]
num_rejected: torch.Tensor # [num_reqs]
idx_mapping: torch.Tensor # [num_reqs]
idx_mapping_np: np.ndarray # [num_reqs]
# Records which rows need a deferred postprocess (bool).
need_sampled_mask: np.ndarray # [num_reqs]
# Snapshot of slot generation counters at receive time, used to
# detect requests aborted since then.
gen_at_receive_np: np.ndarray # [num_reqs]
def pp_receive(
num_reqs: int, max_sample_len: int = 1
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
pp = get_pp_group()
assert not pp.is_last_rank
def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None:
"""Return a bool array of shape `[input_batch.num_reqs]` marking requests
with outputs that might be needed in a subsequent (decode) step.
Returns None if no sampled outputs are needed in the requests' next step."""
sampled_tokens = torch.empty(
num_reqs, max_sample_len, dtype=torch.int64, device=pp.device
)
torch.distributed.broadcast(sampled_tokens, src=pp.last_rank, group=pp.device_group)
old_computed = input_batch.num_computed_tokens_np
prefill_len = input_batch.prefill_len_np
max_seq_len = input_batch.max_seq_len_np
assert max_seq_len is not None # always populated under PP
# Exclude non-final prefill chunks (they don't produce a sample).
produces_sample = old_computed + input_batch.num_scheduled_tokens >= prefill_len
# Exclude requests that we know are finished.
not_finishing = np.maximum(old_computed, prefill_len) + 1 < max_seq_len
need_sampled_mask = produces_sample & not_finishing
return need_sampled_mask if need_sampled_mask.any() else None
combined = torch.empty(2, num_reqs, dtype=torch.int32, device=pp.device)
torch.distributed.broadcast(combined, src=pp.last_rank, group=pp.device_group)
num_sampled, num_rejected = combined.unbind(dim=0)
return sampled_tokens, num_sampled, num_rejected
class PPHandler:
"""Runs the PP sampled-token broadcast/recv on a side stream so the
default stream isn't gated by the matching peer call. Step T's recv is
consumed at step T+pp_size via `get_prev_sampled_outputs`.
Uses a dedicated NCCL communicator (sibling of the PP `device_group`)
for the broadcast so it does not serialize on the wire with the
inter-stage hidden-state p2p send/recv ops.
"""
def __init__(
self, max_num_reqs: int, num_speculative_steps: int, device: torch.device
):
self.is_last_rank = get_pp_group().is_last_rank
self.last_rank = get_pp_group().last_rank
self.max_sample_len = num_speculative_steps + 1
self.device = device
self.main_stream = torch.cuda.current_stream(device)
self.broadcast_stream = torch.cuda.Stream(device)
# On non-last ranks, a FIFO with one entry per in-flight step: the entry
# pushed by step T's `receive` is consumed pp_size steps later. Pre-seeded
# with pp_size None placeholders so the first pp_size consumes are no-ops.
# None means no postprocess is pending for that step (broadcast skipped).
self.queue: deque[PendingRecv | None] = (
deque() if self.is_last_rank else deque([None] * get_pp_group().world_size)
)
# Per req-index generation counter, incremented every time a request
# index is freed in RequestStats. Used for invalidating freed req data
# between PP decodes.
self.req_idx_gen_np = np.zeros(max_num_reqs, dtype=np.int32)
# Dedicated subgroup for the sampled-token broadcast.
self.broadcast_group = get_pp_group().make_sibling_device_group(
group_desc="pp_broadcast"
)
def on_req_idx_freed(self, req_idx: int) -> None:
self.req_idx_gen_np[req_idx] += 1
def get_prev_sampled_outputs(self) -> dict[str, torch.Tensor] | None:
"""Consume the entry from pp_size steps ago and wait for its recv event,
then filter out entries whose request was freed since `receive`.
"""
if not self.queue:
return None
slot = self.queue.popleft()
# Reserve this step's slot; `receive` overwrites it if applicable.
self.queue.append(None)
if slot is None:
return None
# Skip requests which did not need sampled output and/or those already
# finished. The post_update kernel skips the -1 entries.
freed = self.req_idx_gen_np[slot.idx_mapping_np] != slot.gen_at_receive_np
exclude_mask = freed | ~slot.need_sampled_mask
idx_mapping = slot.idx_mapping
if exclude_mask.any():
if exclude_mask.all():
# No states require update anymore.
return None
# Filter excluded request indices.
idx_mapping_np = np.where(exclude_mask, -1, slot.idx_mapping_np)
idx_mapping = async_copy_to_gpu(idx_mapping_np, device=self.device)
self.main_stream.wait_event(slot.event)
return dict(
sampled_tokens=slot.sampled_tokens,
num_sampled=slot.num_sampled,
num_rejected=slot.num_rejected,
idx_mapping=idx_mapping,
)
def receive(self, input_batch: InputBatch) -> bool:
"""Returns True iff sampled tokens need to be gathered from *all*
requests in the batch."""
assert not self.is_last_rank
need_sampled_mask = compute_need_sampled_mask(input_batch)
if need_sampled_mask is None:
# Leave this step's reserved slot as None.
return False
# Snapshot the per-slot generation counter so a later free of any of
# these RequestStates request indices is detectable at consume time.
gen_at_receive_np = self.req_idx_gen_np[input_batch.idx_mapping_np]
num_reqs = input_batch.num_reqs
with torch.cuda.stream(self.broadcast_stream):
self.broadcast_stream.wait_stream(self.main_stream)
sampled_tokens = torch.empty(
num_reqs, self.max_sample_len, dtype=torch.int64, device=self.device
)
combined = torch.empty(2, num_reqs, dtype=torch.int32, device=self.device)
torch.distributed.broadcast(
sampled_tokens, src=self.last_rank, group=self.broadcast_group
)
torch.distributed.broadcast(
combined, src=self.last_rank, group=self.broadcast_group
)
event = self.broadcast_stream.record_event()
num_sampled, num_rejected = combined.unbind(dim=0)
# Must record_stream since these were allocated on broadcast stream but
# later used on the main stream.
sampled_tokens.record_stream(self.main_stream)
combined.record_stream(self.main_stream)
self.queue[-1] = PendingRecv(
event,
sampled_tokens,
num_sampled,
num_rejected,
input_batch.idx_mapping,
input_batch.idx_mapping_np,
need_sampled_mask,
gen_at_receive_np,
)
return bool(need_sampled_mask.all())
def broadcast(
self,
sampled_token_ids: torch.Tensor,
num_sampled: torch.Tensor,
num_rejected: torch.Tensor,
input_batch: InputBatch,
) -> None:
assert self.is_last_rank
if compute_need_sampled_mask(input_batch) is None:
# No request needs sampled outputs for a subsequent decode step.
return
assert sampled_token_ids.dtype == torch.int64
with torch.cuda.stream(self.broadcast_stream):
self.broadcast_stream.wait_stream(self.main_stream)
torch.distributed.broadcast(
sampled_token_ids.contiguous(),
src=self.last_rank,
group=self.broadcast_group,
)
combined = torch.stack((num_sampled, num_rejected), dim=0)
torch.distributed.broadcast(
combined, src=self.last_rank, group=self.broadcast_group
)
for tensor in (sampled_token_ids, num_sampled, num_rejected):
tensor.record_stream(self.broadcast_stream)
+2 -6
View File
@@ -42,10 +42,6 @@ class PromptLogprobsWorker:
num_computed_tokens: torch.Tensor,
# [max_num_reqs]
prompt_lens: np.ndarray,
# [max_num_reqs]
prefill_lens: np.ndarray,
# [max_num_reqs]
num_computed_prefill_tokens: np.ndarray,
) -> dict[str, LogprobsTensors]:
idx_mapping_np = input_batch.idx_mapping_np
needs_prompt_logprobs = self.uses_prompt_logprobs[idx_mapping_np]
@@ -55,11 +51,11 @@ class PromptLogprobsWorker:
num_prompt_logprobs = self.num_prompt_logprobs[idx_mapping_np]
prompt_lens = prompt_lens[idx_mapping_np]
computed_prefill = num_computed_prefill_tokens[idx_mapping_np]
computed_prefill = input_batch.num_computed_prefill_tokens_np
includes_prompt = computed_prefill < prompt_lens
# NOTE(woosuk): If the request was resumed after preemption, its prompt
# logprobs must have been computed before preemption. Skip.
resumed_after_prompt = prompt_lens < prefill_lens[idx_mapping_np]
resumed_after_prompt = prompt_lens < input_batch.prefill_len_np
needs_prompt_logprobs &= includes_prompt & ~resumed_after_prompt
if not np.any(needs_prompt_logprobs):
return {}
+9 -10
View File
@@ -65,6 +65,9 @@ class RequestState:
self.max_num_reqs, 1, dtype=torch.int64, device=device
)
# Max total seq length (prompt_len + max_tokens).
self.max_seq_len = np.zeros(self.max_num_reqs, dtype=np.int32)
# Draft tokens.
self.draft_tokens = torch.zeros(
self.max_num_reqs,
@@ -87,12 +90,14 @@ class RequestState:
prompt_len: int,
all_token_ids: list[int],
num_computed_tokens: int,
max_tokens: int,
) -> None:
assert len(self.free_indices) > 0, "No free indices"
req_idx = self.free_indices.pop()
self.req_id_to_index[req_id] = req_idx
self.index_to_req_id[req_idx] = req_id
self.max_seq_len[req_idx] = prompt_len + max_tokens
self.prompt_len.np[req_idx] = prompt_len
prefill_len = len(all_token_ids)
assert prefill_len >= prompt_len, (
@@ -124,17 +129,11 @@ class RequestState:
self.all_token_ids.apply_write()
self.num_computed_tokens.apply_write()
def remove_request(self, req_id: str) -> bool:
def remove_request(self, req_id: str) -> int | None:
"""Return the freed slot index, or None if the request was not found."""
req_idx = self.req_id_to_index.pop(req_id, None)
if req_idx is None:
# Request not found.
return False
return None
self.index_to_req_id.pop(req_idx, None)
self.free_indices.append(req_idx)
return True
def is_prefilling(self, idx_mapping_np: np.ndarray) -> np.ndarray:
return (
self.num_computed_prefill_tokens[idx_mapping_np]
< self.prefill_len.np[idx_mapping_np]
)
return req_idx