[Model Runner V2] Fix slot_mapping after #25954 (#33046)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Woosuk Kwon
2026-01-25 18:29:49 -08:00
committed by GitHub
co-authored by Woosuk Kwon
parent 22aeb43007
commit edf927bc9f
5 changed files with 40 additions and 23 deletions
+16 -4
View File
@@ -13,7 +13,10 @@ from vllm.model_executor.model_loader import get_model
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backend import AttentionMetadataBuilder
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import build_attn_metadata
from vllm.v1.worker.gpu.attn_utils import (
build_attn_metadata,
build_slot_mappings_by_layer,
)
from vllm.v1.worker.gpu.block_table import BlockTables
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
@@ -108,7 +111,8 @@ class EagleSpeculator:
def run_model(
self,
num_tokens: int,
attn_metadata: dict[str, Any],
attn_metadata: dict[str, Any] | None,
slot_mappings: dict[str, torch.Tensor] | None,
num_tokens_across_dp: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
with set_forward_context(
@@ -117,6 +121,7 @@ class EagleSpeculator:
num_tokens=num_tokens,
cudagraph_runtime_mode=CUDAGraphMode.NONE,
num_tokens_across_dp=num_tokens_across_dp,
slot_mapping=slot_mappings,
):
ret_hidden_states = self.model(
input_ids=self.input_buffers.input_ids[:num_tokens],
@@ -134,6 +139,7 @@ class EagleSpeculator:
self,
num_reqs: int,
attn_metadata: dict[str, Any],
slot_mappings: dict[str, torch.Tensor],
num_tokens_across_dp: torch.Tensor | None,
) -> None:
pos = self.input_buffers.positions[:num_reqs]
@@ -142,7 +148,7 @@ class EagleSpeculator:
for step in range(1, self.num_speculative_steps):
# Run the eagle model.
last_hidden_states, hidden_states = self.run_model(
num_reqs, attn_metadata, num_tokens_across_dp
num_reqs, attn_metadata, slot_mappings, num_tokens_across_dp
)
logits = self.model.compute_logits(last_hidden_states)
@@ -235,6 +241,7 @@ class EagleSpeculator:
last_hidden_states, hidden_states = self.run_model(
num_tokens,
input_batch.attn_metadata,
input_batch.slot_mappings,
num_tokens_across_dp=None, # FIXME
)
sample_hidden_states = last_hidden_states[last_token_indices]
@@ -311,7 +318,12 @@ class EagleSpeculator:
slot_mappings=slot_mappings,
kv_cache_config=self.kv_cache_config,
)
self.generate_draft(num_reqs, attn_metadata, num_tokens_across_dp=None) # FIXME
slot_mappings_by_layer = build_slot_mappings_by_layer(
slot_mappings, self.kv_cache_config
)
self.generate_draft(
num_reqs, attn_metadata, slot_mappings_by_layer, num_tokens_across_dp=None
) # FIXME
return self.draft_tokens[:num_reqs]
@@ -69,7 +69,7 @@ class EagleCudaGraphManager:
kv_cache_config: KVCacheConfig,
) -> None:
num_reqs = min(num_tokens, self.max_num_reqs)
attn_metadata = prepare_inputs_to_capture(
attn_metadata, slot_mappings = prepare_inputs_to_capture(
num_reqs,
num_tokens,
input_buffers,
@@ -81,13 +81,13 @@ class EagleCudaGraphManager:
num_tokens_across_dp = make_num_tokens_across_dp(self.dp_size, num_tokens)
# Warm up.
generate_fn(num_tokens, attn_metadata, num_tokens_across_dp)
generate_fn(num_tokens, attn_metadata, slot_mappings, num_tokens_across_dp)
# Capture the graph.
assert num_tokens not in self.graphs
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, self.pool):
generate_fn(num_tokens, attn_metadata, num_tokens_across_dp)
generate_fn(num_tokens, attn_metadata, slot_mappings, num_tokens_across_dp)
self.graphs[num_tokens] = graph
@torch.inference_mode()