[Perf][GLM-5.2] Blackwell decode optimizations (#48597)

Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com>
Signed-off-by: zhou <zhou@zhoudeMacBook-Pro.local>
Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: zhou <zhou@zhoudeMacBook-Pro.local>
Co-authored-by: stefankoncarevic <Stefan.Koncarevic@amd.com>
This commit is contained in:
xiaozhoupy
2026-07-23 21:36:27 -07:00
committed by GitHub
co-authored by Claude Giancarlo Delfin Thien Tran OpenAI Codex zhou stefankoncarevic
parent 0d77325b10
commit dd72658e7d
29 changed files with 2139 additions and 138 deletions
@@ -43,7 +43,7 @@ try:
if flashinfer_comm is not None
else None
)
except ImportError:
except (ImportError, AttributeError):
flashinfer_trtllm_fused_allreduce_norm = None # type: ignore[assignment]
get_fi_ar_workspace = None # type: ignore[assignment]
_AR_RESIDUAL_RMS_NORM = None
@@ -52,12 +52,23 @@ except ImportError:
_FI_SUPPORTED_DTYPES = (torch.bfloat16, torch.float16)
@torch.compiler.assume_constant_result
def _fi_ar_max_size_mb() -> dict[int, float]:
"""Flashinfer all-reduce fusion size table for the current device.
Device capability is constant; marking the result constant keeps this out
of the traced graph. Otherwise the per-forward call reaches
``current_platform.get_device_capability()`` inside ``torch.compile`` and
graph-breaks ("can't handle functions not implemented in python")."""
from vllm.config.compilation import PassConfig
return PassConfig.default_fi_allreduce_fusion_max_size_mb()
def _max_token_num(tp_size: int, hidden_size: int, dtype: torch.dtype) -> int | None:
"""Workspace token budget for flashinfer fused all-reduce, or None if the
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``."""
from vllm.config.compilation import PassConfig
max_size_mb = PassConfig.default_fi_allreduce_fusion_max_size_mb().get(tp_size)
max_size_mb = _fi_ar_max_size_mb().get(tp_size)
if not max_size_mb:
return None
element_size = torch.tensor([], dtype=dtype).element_size()
+2 -1
View File
@@ -113,7 +113,7 @@ _TEXT_GENERATION_MODELS = {
"Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"),
"Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"),
"Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"),
"GlmMoeDsaForCausalLM": ("deepseek_v2", "GlmMoeDsaForCausalLM"),
"GlmMoeDsaForCausalLM": ("vllm.models.deepseek_v32", "GlmMoeDsaForCausalLM"),
"GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
"GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
"GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
@@ -628,6 +628,7 @@ _SPECULATIVE_DECODING_MODELS = {
"Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"),
"EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"),
"DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"),
"DeepseekV32MTPModel": ("vllm.models.deepseek_v32", "DeepseekV32MTP"),
"DeepSeekV4MTPModel": ("vllm.models.deepseek_v4", "DeepSeekV4MTP"),
"MiniMaxM3MTP": ("vllm.models.minimax_m3", "MiniMaxM3MTP"),
"BailingMoeV25MTPModel": ("bailing_moe_mtp", "BailingMoeV25MTPModel"),