[Perf][GLM-5.2] Blackwell decode optimizations (#48597)

Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com>
Signed-off-by: zhou <zhou@zhoudeMacBook-Pro.local>
Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: zhou <zhou@zhoudeMacBook-Pro.local>
Co-authored-by: stefankoncarevic <Stefan.Koncarevic@amd.com>
This commit is contained in:
xiaozhoupy
2026-07-23 21:36:27 -07:00
committed by GitHub
co-authored by Claude Giancarlo Delfin Thien Tran OpenAI Codex zhou stefankoncarevic
parent 0d77325b10
commit dd72658e7d
29 changed files with 2139 additions and 138 deletions
@@ -48,6 +48,18 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
self.prefill_cudagraph_manager: SpeculatorCudaGraphManager | None = None
self.decode_cudagraph_manager: SpeculatorCudaGraphManager | None = None
# Lifecycle hooks for model-specific optimizations. Subclasses override
# the ones they need. These fire in both `capture` and `propose` so that
# any state they toggle (e.g. attention flags baked into a CUDA graph) is
# identical at capture time and replay time.
def on_prefill_begin(self, num_reqs: int, num_tokens: int) -> None: ...
def on_prefill_end(self, num_reqs: int, num_tokens: int) -> None: ...
def on_multi_step_decode_begin(self, num_reqs: int) -> None: ...
def on_multi_step_decode_end(self, num_reqs: int) -> None: ...
@property
def advance_draft_positions(self) -> bool:
"""
@@ -98,6 +110,9 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
assert self.prefill_cudagraph_manager is not None
if self.prefill_cudagraph_manager.use_breakable_cg:
self.prefill_cudagraph_manager.init_breakable_cg_runner(self.model)
self.on_prefill_begin(self.max_num_reqs, self.max_num_tokens)
self.prefill_cudagraph_manager.capture(
self._prefill,
self.model_state,
@@ -108,9 +123,13 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
progress_bar_desc="Capturing prefill CUDA graphs",
)
self.on_prefill_end(self.max_num_reqs, self.max_num_tokens)
if self.num_speculative_steps == 1:
return
self.on_multi_step_decode_begin(self.max_num_reqs)
# Capture the decode draft generation routine (model forward +
# sample + update_draft_inputs) for a single
# step.
@@ -125,6 +144,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
progress_bar_desc="Capturing decode CUDA graphs",
)
self.on_multi_step_decode_end(self.max_num_reqs)
@torch.inference_mode()
def propose(
self,
@@ -153,7 +174,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None,
is_profile: bool = False,
) -> torch.Tensor:
num_tokens = input_batch.num_tokens_after_padding
num_tokens = input_batch.num_tokens
num_tokens_padded = input_batch.num_tokens_after_padding
num_reqs = input_batch.num_reqs
max_query_len = input_batch.num_scheduled_tokens.max()
max_seq_len = input_batch.seq_lens_cpu_upper_bound[:num_reqs].max().item()
@@ -174,7 +196,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
)
else:
hidden_states = last_hidden_states
self.hidden_states[:num_tokens].copy_(hidden_states)
self.hidden_states[:num_tokens_padded].copy_(hidden_states)
self._copy_request_inputs(
num_reqs,
@@ -202,20 +224,22 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
num_reqs,
# Use the actual number of tokens without padding added by
# the target model during FULL cudagraph.
input_batch.num_tokens,
num_tokens,
max_query_len,
)
prefill_batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp(
self.prefill_cudagraph_manager,
num_reqs,
num_tokens,
num_tokens_padded,
uniform_token_count,
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
)
self._prepare_eplb_forward(input_batch.num_tokens)
self._prepare_eplb_forward(num_tokens)
self.on_prefill_begin(num_reqs, num_tokens)
if prefill_batch_desc.cg_mode == CUDAGraphMode.FULL:
# Replay the full graph for draft prefill.
@@ -235,6 +259,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
mm_inputs=mm_inputs,
)
self.on_prefill_end(num_reqs, num_tokens)
if self.num_speculative_steps == 1:
# Early exit.
return self.draft_tokens[:num_reqs, :1]
@@ -262,6 +288,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
need_eager=is_profile,
)
self.on_multi_step_decode_begin(num_reqs)
# Generate the remaining num_speculative_steps - 1 draft tokens.
self._multi_step_decode(
num_reqs,
@@ -271,6 +299,8 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
input_batch.seq_lens_cpu_upper_bound,
)
self.on_multi_step_decode_end(num_reqs)
return self.draft_tokens[:num_reqs]
@torch.inference_mode()
@@ -1,8 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
)
@@ -10,9 +12,46 @@ from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
class MTPSpeculator(AutoRegressiveSpeculator):
def __init__(self, vllm_config: VllmConfig, device: torch.device):
super().__init__(vllm_config, device)
spec_config = vllm_config.speculative_config
draft_hf_config = (
spec_config.draft_model_config.hf_config
if spec_config is not None
else None
)
# Detect index_share_for_mtp_iteration. When True, the proposer
# toggles skip_topk so step 0 computes MTP's own indices and
# steps 1+ reuse them.
self.share_mtp_topk_indices = getattr(
draft_hf_config, "index_share_for_mtp_iteration", False
)
def load_draft_model(
self,
target_model: nn.Module,
target_attn_layer_names: set[str],
) -> nn.Module:
return load_eagle_model(target_model, self.vllm_config)
draft_model = load_eagle_model(target_model, self.vllm_config)
self.share_mtp_topk_indices = self.share_mtp_topk_indices and hasattr(
draft_model.model, "set_skip_topk"
)
return draft_model
def on_prefill_end(self, num_reqs: int, num_tokens: int) -> None:
# Step 0 (prefill) wrote topk indices for every query token in the
# multi-token batch. Compact them down to each request's last token so
# steps 1+ can reuse them from the shared buffer.
if self.share_mtp_topk_indices and self.num_speculative_steps > 1:
self.model.model.compact_topk_indices(self.last_token_indices[:num_reqs])
def on_multi_step_decode_begin(self, num_reqs: int) -> None:
# Switch to reuse mode so draft steps 1+ skip the indexer op and read
# the indices that step 0 wrote into the shared buffer.
if self.share_mtp_topk_indices:
self.model.model.set_skip_topk(True)
def on_multi_step_decode_end(self, num_reqs: int) -> None:
if self.share_mtp_topk_indices:
self.model.model.set_skip_topk(False)