[Model Runner V2] Support VLM (#32546)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
Woosuk Kwon
2026-01-18 16:58:51 -08:00
committed by GitHub
parent 6101a26dc9
commit bb1848cd62
6 changed files with 263 additions and 15 deletions
+3 -3
View File
@@ -15,9 +15,6 @@ class InputBuffers:
self,
max_num_reqs: int,
max_num_tokens: int,
inputs_embeds_size: int,
vocab_size: int,
dtype: torch.dtype,
device: torch.device,
):
self.max_num_reqs = max_num_reqs
@@ -64,6 +61,8 @@ class InputBatch:
positions: torch.Tensor
# [3, num_tokens_after_padding]
mrope_positions: torch.Tensor | None
# [num_tokens_after_padding, hidden_size]
inputs_embeds: torch.Tensor | None
# layer_name -> Metadata
attn_metadata: dict[str, Any]
@@ -132,6 +131,7 @@ class InputBatch:
input_ids=input_ids,
positions=positions,
mrope_positions=None,
inputs_embeds=None,
attn_metadata=None, # type: ignore
logits_indices=logits_indices,
cu_num_logits=cu_num_logits,