[Model Runner V2] Support Streaming Inputs (#37028)

Signed-off-by: Santino Ramos <elsantinoramos@gmail.com>
This commit is contained in:
Santino Ramos
2026-03-20 20:42:25 +00:00
committed by GitHub
parent 8bc6b5cdb0
commit 85f671b8e1
7 changed files with 263 additions and 12 deletions
+6 -1
View File
@@ -111,6 +111,7 @@ def _reshape_kv_cache(
kv_cache_config: KVCacheConfig,
kv_cache_raw_tensors: dict[str, torch.Tensor],
attn_backends: dict[str, AttentionBackend],
cache_dtype: str,
) -> dict[str, torch.Tensor]:
kv_caches: dict[str, torch.Tensor] = {}
for kv_cache_group_spec in kv_cache_config.kv_cache_groups:
@@ -127,6 +128,7 @@ def _reshape_kv_cache(
kv_cache_spec.block_size,
kv_cache_spec.num_kv_heads,
kv_cache_spec.head_size,
cache_dtype,
)
# FIXME(woosuk): Add kv_cache_stride_order to all attention backends.
@@ -155,9 +157,12 @@ def init_kv_cache(
kv_cache_config: KVCacheConfig,
attn_backends: dict[str, AttentionBackend],
device: torch.device,
cache_dtype: str,
) -> dict[str, torch.Tensor]:
kv_cache_raw_tensors = _allocate_kv_cache(kv_cache_config, device)
kv_caches = _reshape_kv_cache(kv_cache_config, kv_cache_raw_tensors, attn_backends)
kv_caches = _reshape_kv_cache(
kv_cache_config, kv_cache_raw_tensors, attn_backends, cache_dtype
)
bind_kv_cache(kv_caches, forward_context, runner_kv_caches)
return kv_caches
+18 -6
View File
@@ -359,6 +359,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.kv_cache_config,
self.attn_backends,
self.device,
self.cache_config.cache_dtype,
)
self.kv_connector = get_kv_connector(self.vllm_config, kv_caches_dict)
@@ -555,18 +556,23 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
return cuda_graph_size
def _remove_request(self, req_id: str) -> bool:
if not self.req_states.remove_request(req_id):
return False
if self.encoder_cache is not None:
self.encoder_cache.remove_request(req_id)
if self.prompt_logprobs_worker is not None:
self.prompt_logprobs_worker.remove_request(req_id)
self.lora_state.remove_request(req_id)
return True
def finish_requests(self, scheduler_output: SchedulerOutput) -> None:
finished_req_ids = scheduler_output.finished_req_ids
preempted_req_ids = scheduler_output.preempted_req_ids
if preempted_req_ids:
finished_req_ids = finished_req_ids.union(preempted_req_ids)
for req_id in finished_req_ids:
self.req_states.remove_request(req_id)
if self.encoder_cache is not None:
self.encoder_cache.remove_request(req_id)
if self.prompt_logprobs_worker is not None:
self.prompt_logprobs_worker.remove_request(req_id)
self.lora_state.remove_request(req_id)
self._remove_request(req_id)
def free_states(self, scheduler_output: SchedulerOutput) -> None:
if self.encoder_cache is not None:
@@ -578,6 +584,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
assert new_req_data.prompt_token_ids is not None
assert new_req_data.prefill_token_ids is not None
req_id = new_req_data.req_id
# Streaming input update: request already exists from a prior
# chunk. Remove old state so it can be cleanly re-added below
# with the updated prompt_token_ids and mm_features.
self._remove_request(req_id)
prompt_len = len(new_req_data.prompt_token_ids)
self.req_states.add_request(
req_id=req_id,
@@ -7,6 +7,7 @@ import torch.nn as nn
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.tasks import GenerationTask
from vllm.v1.core.sched.output import NewRequestData
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
@@ -61,6 +62,28 @@ class DefaultModelState(ModelState):
device=self.device,
)
def get_supported_generation_tasks(self) -> tuple[GenerationTask, ...]:
from vllm.model_executor.models.interfaces import (
supports_realtime,
supports_transcription,
)
from vllm.model_executor.models.interfaces_base import is_text_generation_model
supported_tasks = list[GenerationTask]()
if is_text_generation_model(self.model):
supported_tasks.append("generate")
if supports_transcription(self.model):
if self.model.supports_transcription_only:
return ("transcription",)
supported_tasks.append("transcription")
if supports_realtime(self.model):
supported_tasks.append("realtime")
return tuple(supported_tasks)
def add_request(self, req_index: int, new_req_data: NewRequestData) -> None:
if self.rope_state is not None:
assert new_req_data.prefill_token_ids is not None
+2 -1
View File
@@ -28,8 +28,9 @@ class ModelState(ABC):
) -> None:
raise NotImplementedError
@abstractmethod
def get_supported_generation_tasks(self) -> tuple[GenerationTask, ...]:
return ("generate",)
raise NotImplementedError
def add_request(self, req_index: int, new_req_data: NewRequestData) -> None:
return None
+3 -2
View File
@@ -109,13 +109,14 @@ class RequestState:
self.all_token_ids.apply_write()
self.num_computed_tokens.apply_write()
def remove_request(self, req_id: str) -> None:
def remove_request(self, req_id: str) -> bool:
req_idx = self.req_id_to_index.pop(req_id, None)
if req_idx is None:
# Request not found.
return
return False
self.index_to_req_id.pop(req_idx, None)
self.free_indices.append(req_idx)
return True
def any_prefills(self, idx_mapping_np: np.ndarray) -> bool:
return np.any(