[Perf][GLM-5.2] Blackwell decode optimizations (#48597)
Signed-off-by: Peiyuan Zhou <peiyuanzhou1994@gmail.com> Signed-off-by: zhou <zhou@zhoudeMacBook-Pro.local> Signed-off-by: Stefan Koncarevic <stefan.koncarevic@amd.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: zhou <zhou@zhoudeMacBook-Pro.local> Co-authored-by: stefankoncarevic <Stefan.Koncarevic@amd.com>
This commit is contained in:
co-authored by
Claude
Giancarlo Delfin
Thien Tran
OpenAI Codex
zhou
stefankoncarevic
parent
0d77325b10
commit
dd72658e7d
@@ -202,11 +202,7 @@ if flashinfer_comm is not None:
|
||||
element_size = allreduce_in.element_size()
|
||||
current_tensor_size = num_tokens * hidden_size * element_size
|
||||
max_tensor_size = max_token_num * hidden_size * element_size
|
||||
assert current_tensor_size <= max_tensor_size, (
|
||||
f"Current tensor size {current_tensor_size} is larger than "
|
||||
f"max token num {max_token_num} * hidden size {hidden_size} * "
|
||||
f"element size {element_size}"
|
||||
)
|
||||
oversize = current_tensor_size > max_tensor_size
|
||||
curr_device = current_platform.get_device_capability()
|
||||
device_capability = curr_device.to_int() if curr_device is not None else None
|
||||
|
||||
@@ -219,6 +215,47 @@ if flashinfer_comm is not None:
|
||||
get_workspace_fn = (
|
||||
get_fi_ar_quant_workspace if is_quant_pattern else get_fi_ar_workspace
|
||||
)
|
||||
|
||||
orig_norm_out = norm_out
|
||||
|
||||
def unfused_fallback() -> None:
|
||||
# Fused-path equivalent honoring the op's output-buffer contract:
|
||||
# with norm_out given, normed -> norm_out and the new residual ->
|
||||
# the allreduce_in buffer; otherwise normed -> allreduce_in and
|
||||
# the new residual -> the residual buffer. Runs fine during CUDA
|
||||
# graph capture (the branch is resolved per captured shape).
|
||||
#
|
||||
# weight_bias mirrors the fused kernel's gamma offset (Gemma
|
||||
# patterns pass 1.0 for rms_norm(x, weight + 1)); fold it into
|
||||
# the gamma in fp32 like the reference before the plain rms_norm.
|
||||
gamma = (
|
||||
rms_gamma
|
||||
if weight_bias == 0.0
|
||||
else (rms_gamma.float() + weight_bias).to(rms_gamma.dtype)
|
||||
)
|
||||
reduced = tensor_model_parallel_all_reduce(allreduce_in)
|
||||
new_residual = reduced + residual
|
||||
if orig_norm_out is None:
|
||||
torch.ops._C.rms_norm(allreduce_in, new_residual, gamma, rms_eps)
|
||||
residual.copy_(new_residual)
|
||||
else:
|
||||
torch.ops._C.rms_norm(orig_norm_out, new_residual, gamma, rms_eps)
|
||||
allreduce_in.copy_(new_residual)
|
||||
|
||||
if oversize:
|
||||
assert not is_quant_pattern, (
|
||||
"flashinfer fused allreduce: input exceeds the workspace budget "
|
||||
"and the unfused fallback does not cover quant patterns"
|
||||
)
|
||||
logger.warning_once(
|
||||
"fused allreduce+rmsnorm: %d tokens exceeds the workspace "
|
||||
"budget (max_token_num=%d); using the unfused fallback for "
|
||||
"this shape.",
|
||||
num_tokens,
|
||||
max_token_num,
|
||||
)
|
||||
unfused_fallback()
|
||||
return
|
||||
workspace = get_workspace_fn(
|
||||
world_size=world_size,
|
||||
rank=get_tensor_model_parallel_rank(),
|
||||
@@ -252,36 +289,75 @@ if flashinfer_comm is not None:
|
||||
if workspace.backend in ("trtllm", "mnnvl"):
|
||||
layout_code = flashinfer_comm.QuantizationSFLayout.SWIZZLED_128x4
|
||||
|
||||
flashinfer_comm.allreduce_fusion(
|
||||
input=allreduce_in,
|
||||
workspace=workspace,
|
||||
pattern=pattern_code,
|
||||
launch_with_pdl=launch_with_pdl,
|
||||
output=None,
|
||||
residual_out=residual_out,
|
||||
norm_out=norm_out,
|
||||
quant_out=quant_out,
|
||||
scale_out=scale_out,
|
||||
residual_in=residual,
|
||||
rms_gamma=rms_gamma,
|
||||
rms_eps=rms_eps,
|
||||
scale_factor=scale_factor,
|
||||
layout_code=layout_code,
|
||||
use_oneshot=use_oneshot,
|
||||
fp32_acc=fp32_acc,
|
||||
weight_bias=weight_bias,
|
||||
# The one-shot Lamport all-reduce signals PDL completion before its
|
||||
# output buffer is committed when trigger_completion_at_end is
|
||||
# False, so the next PDL-launched kernel can read the uninitialized
|
||||
# Lamport buffer and produce NaN. This only fires for
|
||||
# num_tokens <= PDL_ADVANCE_LAUNCH_TOKENS (the batch=1 / spec-decode
|
||||
# shapes, where the one-shot path is always selected). Complete at
|
||||
# the end for the one-shot path; the two-shot path is synchronized
|
||||
# and keeps the early completion. Related one-shot instability in
|
||||
# the same kernel: flashinfer-ai/flashinfer#1223.
|
||||
trigger_completion_at_end=(use_oneshot is True)
|
||||
or num_tokens > PDL_ADVANCE_LAUNCH_TOKENS,
|
||||
)
|
||||
# vllm's one-shot size table is tuned for the trtllm backend. For
|
||||
# mnnvl, flashinfer caps the workspace's one-shot region at its own
|
||||
# MNNVL_ONE_SHOT_THRESHOLD (~4MB, e.g. 44 tokens at hidden 6144 bf16
|
||||
# TP8); forcing use_oneshot=True past that raises ValueError instead
|
||||
# of using two-shot, so every decode capture between the one-shot cap
|
||||
# and the fusion threshold silently lost fusion. Let flashinfer's
|
||||
# AUTO heuristic pick the strategy (it matches its own workspace
|
||||
# sizing), and keep trigger_completion_at_end=True since AUTO may
|
||||
# select one-shot (see the Lamport NaN note below).
|
||||
mnnvl_auto = workspace.backend == "mnnvl"
|
||||
if mnnvl_auto:
|
||||
use_oneshot = None
|
||||
|
||||
try:
|
||||
flashinfer_comm.allreduce_fusion(
|
||||
input=allreduce_in,
|
||||
workspace=workspace,
|
||||
pattern=pattern_code,
|
||||
launch_with_pdl=launch_with_pdl,
|
||||
output=None,
|
||||
residual_out=residual_out,
|
||||
norm_out=norm_out,
|
||||
quant_out=quant_out,
|
||||
scale_out=scale_out,
|
||||
residual_in=residual,
|
||||
rms_gamma=rms_gamma,
|
||||
rms_eps=rms_eps,
|
||||
scale_factor=scale_factor,
|
||||
layout_code=layout_code,
|
||||
use_oneshot=use_oneshot,
|
||||
fp32_acc=fp32_acc,
|
||||
weight_bias=weight_bias,
|
||||
# The one-shot Lamport all-reduce signals PDL completion before
|
||||
# its output buffer is committed when trigger_completion_at_end
|
||||
# is False, so the next PDL-launched kernel can read the
|
||||
# uninitialized Lamport buffer and produce NaN. This only fires
|
||||
# for num_tokens <= PDL_ADVANCE_LAUNCH_TOKENS (the batch=1 /
|
||||
# spec-decode shapes, where the one-shot path is always
|
||||
# selected). Complete at the end for the one-shot path; the
|
||||
# two-shot path is synchronized and keeps the early completion.
|
||||
# Related one-shot instability in the same kernel:
|
||||
# flashinfer-ai/flashinfer#1223.
|
||||
trigger_completion_at_end=True
|
||||
if mnnvl_auto
|
||||
else (use_oneshot or num_tokens > PDL_ADVANCE_LAUNCH_TOKENS),
|
||||
)
|
||||
except ValueError as e:
|
||||
# The globally cached workspace can be smaller than this call's
|
||||
# budget (it is created once by whichever caller runs first, and
|
||||
# the mnnvl workspace's is_buffer_size_sufficient() checks trtllm
|
||||
# metadata, so vllm-side accounting cannot see the real capacity).
|
||||
# flashinfer raises before launching anything, so falling back
|
||||
# mid-capture is safe. Observed with spec-decode draft graphs
|
||||
# whose token count (max_num_reqs * (1 + num_spec)) exceeds the
|
||||
# workspace sized for the fusion threshold.
|
||||
assert not is_quant_pattern, (
|
||||
"flashinfer fused allreduce failed and the unfused fallback "
|
||||
f"does not cover quant patterns: {e}"
|
||||
)
|
||||
logger.warning_once(
|
||||
"flashinfer fused allreduce+rmsnorm unavailable for this "
|
||||
"shape (num_tokens=%d, max_token_num=%d, world_size=%d): %s. "
|
||||
"Using the unfused fallback.",
|
||||
num_tokens,
|
||||
max_token_num,
|
||||
world_size,
|
||||
e,
|
||||
)
|
||||
unfused_fallback()
|
||||
|
||||
def call_trtllm_fused_allreduce_norm_fake(
|
||||
allreduce_in: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user