[Model Runner V2] Move mrope_positions buffer to MRopeState (#32532)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
Woosuk Kwon
2026-01-17 20:09:48 -08:00
committed by GitHub
parent 3055232ba0
commit 4147910f1e
4 changed files with 49 additions and 34 deletions
+7 -4
View File
@@ -75,16 +75,17 @@ class CudaGraphManager:
num_tokens: int,
model: nn.Module,
input_buffers: InputBuffers,
mrope_positions: torch.Tensor | None,
block_tables: BlockTables,
attn_metadata_builders: list[AttentionMetadataBuilder],
kv_cache_config: KVCacheConfig,
) -> None:
num_reqs = min(num_tokens, self.max_num_reqs)
input_ids = input_buffers.input_ids[:num_tokens]
if not self.uses_mrope:
positions = input_buffers.positions[:num_tokens]
else:
positions = input_buffers.mrope_positions[:, :num_tokens]
positions = input_buffers.positions[:num_tokens]
if self.uses_mrope:
assert mrope_positions is not None
positions = mrope_positions[:, :num_tokens]
attn_metadata = prepare_inputs_to_capture(
num_reqs,
num_tokens,
@@ -136,6 +137,7 @@ class CudaGraphManager:
self,
model: nn.Module,
input_buffers: InputBuffers,
mrope_positions: torch.Tensor | None,
block_tables: BlockTables,
attn_metadata_builders: list[AttentionMetadataBuilder],
kv_cache_config: KVCacheConfig,
@@ -146,6 +148,7 @@ class CudaGraphManager:
self.capture_graph,
model=model,
input_buffers=input_buffers,
mrope_positions=mrope_positions,
block_tables=block_tables,
attn_metadata_builders=attn_metadata_builders,
kv_cache_config=kv_cache_config,