[Model Runner V2] Support VLM (#32546)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
Woosuk Kwon
2026-01-18 16:58:51 -08:00
committed by GitHub
parent 6101a26dc9
commit bb1848cd62
6 changed files with 263 additions and 15 deletions
+7
View File
@@ -76,6 +76,7 @@ class CudaGraphManager:
model: nn.Module,
input_buffers: InputBuffers,
mrope_positions: torch.Tensor | None,
inputs_embeds: torch.Tensor | None,
block_tables: BlockTables,
attn_metadata_builders: list[AttentionMetadataBuilder],
kv_cache_config: KVCacheConfig,
@@ -86,6 +87,8 @@ class CudaGraphManager:
if self.uses_mrope:
assert mrope_positions is not None
positions = mrope_positions[:, :num_tokens]
if inputs_embeds is not None:
inputs_embeds = inputs_embeds[:num_tokens]
attn_metadata = prepare_inputs_to_capture(
num_reqs,
num_tokens,
@@ -108,6 +111,7 @@ class CudaGraphManager:
hidden_states = model(
input_ids=input_ids,
positions=positions,
inputs_embeds=inputs_embeds,
)
if self.hidden_states is None:
self.hidden_states = torch.empty_like(hidden_states)
@@ -128,6 +132,7 @@ class CudaGraphManager:
hidden_states = model(
input_ids=input_ids,
positions=positions,
inputs_embeds=inputs_embeds,
)
self.hidden_states[:num_tokens] = hidden_states
self.graphs[num_tokens] = graph
@@ -138,6 +143,7 @@ class CudaGraphManager:
model: nn.Module,
input_buffers: InputBuffers,
mrope_positions: torch.Tensor | None,
inputs_embeds: torch.Tensor | None,
block_tables: BlockTables,
attn_metadata_builders: list[AttentionMetadataBuilder],
kv_cache_config: KVCacheConfig,
@@ -149,6 +155,7 @@ class CudaGraphManager:
model=model,
input_buffers=input_buffers,
mrope_positions=mrope_positions,
inputs_embeds=inputs_embeds,
block_tables=block_tables,
attn_metadata_builders=attn_metadata_builders,
kv_cache_config=kv_cache_config,