[Perf][GLM-5.2] Blackwell decode optimizations (#48597)
Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com> Signed-off-by: zhou <zhou@zhoudeMacBook-Pro.local> Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: zhou <zhou@zhoudeMacBook-Pro.local> Co-authored-by: stefankoncarevic <Stefan.Koncarevic@amd.com>
This commit is contained in:
co-authored by
Claude
Giancarlo Delfin
Thien Tran
OpenAI Codex
zhou
stefankoncarevic
parent
0d77325b10
commit
dd72658e7d
@@ -43,7 +43,7 @@ try:
|
||||
if flashinfer_comm is not None
|
||||
else None
|
||||
)
|
||||
except ImportError:
|
||||
except (ImportError, AttributeError):
|
||||
flashinfer_trtllm_fused_allreduce_norm = None # type: ignore[assignment]
|
||||
get_fi_ar_workspace = None # type: ignore[assignment]
|
||||
_AR_RESIDUAL_RMS_NORM = None
|
||||
@@ -52,12 +52,23 @@ except ImportError:
|
||||
_FI_SUPPORTED_DTYPES = (torch.bfloat16, torch.float16)
|
||||
|
||||
|
||||
@torch.compiler.assume_constant_result
|
||||
def _fi_ar_max_size_mb() -> dict[int, float]:
|
||||
"""Flashinfer all-reduce fusion size table for the current device.
|
||||
|
||||
Device capability is constant; marking the result constant keeps this out
|
||||
of the traced graph. Otherwise the per-forward call reaches
|
||||
``current_platform.get_device_capability()`` inside ``torch.compile`` and
|
||||
graph-breaks ("can't handle functions not implemented in python")."""
|
||||
from vllm.config.compilation import PassConfig
|
||||
|
||||
return PassConfig.default_fi_allreduce_fusion_max_size_mb()
|
||||
|
||||
|
||||
def _max_token_num(tp_size: int, hidden_size: int, dtype: torch.dtype) -> int | None:
|
||||
"""Workspace token budget for flashinfer fused all-reduce, or None if the
|
||||
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``."""
|
||||
from vllm.config.compilation import PassConfig
|
||||
|
||||
max_size_mb = PassConfig.default_fi_allreduce_fusion_max_size_mb().get(tp_size)
|
||||
max_size_mb = _fi_ar_max_size_mb().get(tp_size)
|
||||
if not max_size_mb:
|
||||
return None
|
||||
element_size = torch.tensor([], dtype=dtype).element_size()
|
||||
|
||||
Reference in New Issue
Block a user