[Model Runner V2] Fix slot_mapping after #25954 (#33046)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Woosuk Kwon
2026-01-25 18:29:49 -08:00
committed by GitHub
co-authored by Woosuk Kwon
parent 22aeb43007
commit edf927bc9f
5 changed files with 40 additions and 23 deletions
+7 -7
View File
@@ -101,9 +101,6 @@ class CudaGraphManager:
kv_cache_config,
)
num_tokens_across_dp = make_num_tokens_across_dp(self.dp_size, num_tokens)
slot_mappings_by_layer = build_slot_mappings_by_layer(
slot_mappings, kv_cache_config
)
# Warm up.
with set_forward_context(
@@ -112,7 +109,7 @@ class CudaGraphManager:
num_tokens=num_tokens,
cudagraph_runtime_mode=CUDAGraphMode.NONE,
num_tokens_across_dp=num_tokens_across_dp,
slot_mapping=slot_mappings_by_layer,
slot_mapping=slot_mappings,
):
hidden_states = model(
input_ids=input_ids,
@@ -132,7 +129,7 @@ class CudaGraphManager:
num_tokens=num_tokens,
cudagraph_runtime_mode=CUDAGraphMode.NONE,
num_tokens_across_dp=num_tokens_across_dp,
slot_mapping=slot_mappings_by_layer,
slot_mapping=slot_mappings,
),
torch.cuda.graph(graph, self.pool),
):
@@ -252,7 +249,7 @@ def prepare_inputs_to_capture(
attn_metadata_builders: list[AttentionMetadataBuilder],
max_model_len: int,
kv_cache_config: KVCacheConfig,
) -> tuple[dict[str, Any], torch.Tensor]:
) -> tuple[dict[str, Any], dict[str, torch.Tensor]]:
num_tokens_per_req = num_tokens // num_reqs
query_start_loc_np = np.arange(num_reqs + 1, dtype=np.int32) * num_tokens_per_req
@@ -269,6 +266,9 @@ def prepare_inputs_to_capture(
input_block_tables = [x[:num_reqs] for x in block_tables.input_block_tables]
slot_mappings = block_tables.slot_mappings[:, :num_tokens]
slot_mappings_by_layer = build_slot_mappings_by_layer(
slot_mappings, kv_cache_config
)
attn_metadata = build_attn_metadata(
attn_metadata_builders=attn_metadata_builders,
@@ -282,4 +282,4 @@ def prepare_inputs_to_capture(
slot_mappings=slot_mappings,
kv_cache_config=kv_cache_config,
)
return attn_metadata, slot_mappings
return attn_metadata, slot_mappings_by_layer