[Perf][DSV4.1] Overlap mHC coefficients for small TP batches (#57603)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -305,8 +305,12 @@ def mhc_pre_big_fuse_with_norm_tilelang(
|
||||
use_pre_mix_in: bool = False,
|
||||
save_pre_mix: bool = False,
|
||||
rms_numel: int = 0,
|
||||
split_mode: str = "fused",
|
||||
write_aux: bool = False,
|
||||
):
|
||||
# Split modes require shifted mHC: input collapse uses a carried pre-mix.
|
||||
assert split_mode in ("fused", "stats", "input")
|
||||
assert split_mode == "fused" or save_pre_mix
|
||||
num_tokens = T.dynamic("num_tokens")
|
||||
hc_mult3 = hc_mult * (2 + hc_mult)
|
||||
if gemm_last_dim < 0:
|
||||
@@ -330,26 +334,26 @@ def mhc_pre_big_fuse_with_norm_tilelang(
|
||||
aux_out: T.Tensor[[num_tokens, hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type]
|
||||
|
||||
with T.Kernel(num_tokens, threads=96) as i:
|
||||
rms = T.alloc_fragment(1, T.float32)
|
||||
mixes = T.alloc_fragment(hc_mult3, T.float32)
|
||||
T.clear(mixes)
|
||||
rms[0] = 0
|
||||
|
||||
if ENABLE_PDL:
|
||||
T.pdl_sync()
|
||||
|
||||
for i_split in T.serial(n_splits):
|
||||
rms[0] += gemm_out_sqrsum[i_split, i]
|
||||
rms[0] = T.rsqrt(rms[0] / rms_numel + rms_eps)
|
||||
for j in T.Parallel(hc_mult3):
|
||||
mixes[j] = 0
|
||||
for i_split in T.serial(n_splits):
|
||||
mixes[j] += gemm_out_mul[i_split, i, j]
|
||||
mixes[j] *= rms[0]
|
||||
mixes_shared = T.alloc_shared(hc_mult3, T.float32)
|
||||
T.copy(mixes, mixes_shared)
|
||||
if split_mode != "input":
|
||||
rms = T.alloc_fragment(1, T.float32)
|
||||
mixes = T.alloc_fragment(hc_mult3, T.float32)
|
||||
T.clear(mixes)
|
||||
rms[0] = 0
|
||||
|
||||
if T.get_thread_binding() < 32:
|
||||
for i_split in T.serial(n_splits):
|
||||
rms[0] += gemm_out_sqrsum[i_split, i]
|
||||
rms[0] = T.rsqrt(rms[0] / rms_numel + rms_eps)
|
||||
for j in T.Parallel(hc_mult3):
|
||||
mixes[j] = 0
|
||||
for i_split in T.serial(n_splits):
|
||||
mixes[j] += gemm_out_mul[i_split, i, j]
|
||||
mixes[j] *= rms[0]
|
||||
T.copy(mixes, mixes_shared)
|
||||
|
||||
if split_mode != "input" and T.get_thread_binding() < 32:
|
||||
cm = T.alloc_fragment((hc_mult, hc_mult), T.float32)
|
||||
for j in T.Parallel(hc_mult):
|
||||
if save_pre_mix:
|
||||
@@ -395,7 +399,7 @@ def mhc_pre_big_fuse_with_norm_tilelang(
|
||||
|
||||
for j, k in T.Parallel(hc_mult, hc_mult):
|
||||
comb_mix[i, j * hc_mult + k] = cm[j, k]
|
||||
else:
|
||||
elif split_mode != "stats" and T.get_thread_binding() >= 32:
|
||||
pre_mix_shared = T.alloc_shared(hc_mult, T.float32)
|
||||
for j in T.Parallel(hc_mult):
|
||||
if use_pre_mix_in:
|
||||
|
||||
@@ -36,6 +36,7 @@ class MHCPreNormKernel(VllmJitKernel["MHCPreNormKernel.CompileKey"]):
|
||||
use_pre_mix_in: bool
|
||||
rms_numel: int
|
||||
save_pre_mix: bool = True
|
||||
split_mode: str = "fused"
|
||||
write_aux: bool = False
|
||||
|
||||
kernel: Any = staticmethod(mhc_pre_big_fuse_with_norm_tilelang)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
from functools import partial
|
||||
from itertools import islice
|
||||
|
||||
import regex as re
|
||||
@@ -17,7 +18,11 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.forward_context import (
|
||||
get_forward_context,
|
||||
in_piecewise_cudagraph,
|
||||
is_forward_context_available,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
mhc_post_tilelang,
|
||||
@@ -84,6 +89,11 @@ from ..common.engram import EngramLayout, NgramHashState
|
||||
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
|
||||
from .engram import Engram, gather_engram_hashes
|
||||
from .ops.mega_mhc import mhc_shifted_post_pre
|
||||
from .ops.mhc import (
|
||||
MHC_OVERLAP_MAX_TOKENS,
|
||||
mhc_pre_delayed_overlap,
|
||||
supports_mhc_overlap,
|
||||
)
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
@@ -188,11 +198,13 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
aux_stream_list: list[torch.cuda.Stream] | None = None,
|
||||
candidate_block_buffer: torch.Tensor | None = None,
|
||||
engram_layout: EngramLayout | None = None,
|
||||
mhc_stream: torch.cuda.Stream | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.hidden_size = config.hidden_size
|
||||
self.mhc_stream = mhc_stream
|
||||
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
|
||||
|
||||
self.engram: Engram | None = None
|
||||
@@ -269,19 +281,30 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
variants.append(
|
||||
{**hc_stream, "use_pre_mix_in": True, "write_aux": True}
|
||||
)
|
||||
# Shifted mHC splits the epilogue in two: the input collapse on the
|
||||
# caller stream and coefficient generation on a side stream.
|
||||
split_modes = (
|
||||
("fused", "stats", "input") if mhc_stream is not None else ("fused",)
|
||||
)
|
||||
for variant in variants:
|
||||
MHC_PRE_NORM_KERNEL.register_warmup(
|
||||
max_tokens=max_tokens,
|
||||
hidden_size=self.hidden_size,
|
||||
rms_eps=self.rms_norm_eps,
|
||||
hc_pre_eps=self.hc_eps,
|
||||
hc_sinkhorn_eps=self.hc_eps,
|
||||
hc_post_mult_value=self.hc_post_alpha,
|
||||
sinkhorn_repeat=self.hc_sinkhorn_iters,
|
||||
norm_eps=self.rms_norm_eps,
|
||||
hc_mult=self.hc_mult,
|
||||
**variant,
|
||||
)
|
||||
for split_mode in split_modes:
|
||||
MHC_PRE_NORM_KERNEL.register_warmup(
|
||||
split_mode=split_mode,
|
||||
max_tokens=(
|
||||
max_tokens
|
||||
if split_mode == "fused"
|
||||
else min(max_tokens, MHC_OVERLAP_MAX_TOKENS)
|
||||
),
|
||||
hidden_size=self.hidden_size,
|
||||
rms_eps=self.rms_norm_eps,
|
||||
hc_pre_eps=self.hc_eps,
|
||||
hc_sinkhorn_eps=self.hc_eps,
|
||||
hc_post_mult_value=self.hc_post_alpha,
|
||||
sinkhorn_repeat=self.hc_sinkhorn_iters,
|
||||
norm_eps=self.rms_norm_eps,
|
||||
hc_mult=self.hc_mult,
|
||||
**variant,
|
||||
)
|
||||
mix_hc = (2 + self.hc_mult) * self.hc_mult
|
||||
hc_dim = self.hc_mult * self.hidden_size
|
||||
self.hc_attn_fn = nn.Parameter(
|
||||
@@ -351,6 +374,18 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
torch.Tensor | None,
|
||||
]:
|
||||
previous_aux: torch.Tensor | None = None
|
||||
mhc_stream = self.mhc_stream
|
||||
if mhc_stream is not None and (
|
||||
in_piecewise_cudagraph()
|
||||
or not 0 < positions.shape[0] <= MHC_OVERLAP_MAX_TOKENS
|
||||
):
|
||||
# A side stream cannot remain unjoined across breakable graph segments.
|
||||
mhc_stream = None
|
||||
mhc_pre = (
|
||||
partial(mhc_pre_delayed_overlap, stream=mhc_stream)
|
||||
if mhc_stream is not None
|
||||
else mhc_pre_delayed_tilelang
|
||||
)
|
||||
# The reference collapses each sublayer's input with the *previous*
|
||||
# sublayer's pre-mix: attention uses the pre-mix carried in (identity
|
||||
# for the first layer), the FFN uses this layer's attention pre-mix.
|
||||
@@ -360,7 +395,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
# copies and the identity pre-mix selects copy 0.
|
||||
assert self.hc_attn_fn_broadcast is not None
|
||||
residual = x.unsqueeze(1).expand(-1, self.hc_mult, -1).contiguous()
|
||||
post_mix, res_mix, x, attn_pre = mhc_pre_delayed_tilelang(
|
||||
post_mix, res_mix, x, attn_pre = mhc_pre(
|
||||
residual,
|
||||
self.hc_attn_fn_broadcast,
|
||||
self.hc_attn_scale,
|
||||
@@ -376,7 +411,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
)
|
||||
else:
|
||||
residual = x
|
||||
post_mix, res_mix, x, attn_pre = mhc_pre_delayed_tilelang(
|
||||
post_mix, res_mix, x, attn_pre = mhc_pre(
|
||||
residual,
|
||||
self.hc_attn_fn,
|
||||
self.hc_attn_scale,
|
||||
@@ -403,7 +438,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
engram_hashes[:, self.engram.layer_hash_index],
|
||||
engram_mask,
|
||||
)
|
||||
post_mix, res_mix, x, attn_pre = mhc_pre_delayed_tilelang(
|
||||
post_mix, res_mix, x, attn_pre = mhc_pre(
|
||||
residual,
|
||||
self.hc_attn_fn,
|
||||
self.hc_attn_scale,
|
||||
@@ -437,6 +472,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
norm_weight=self.attn_norm.weight,
|
||||
norm_eps=self.attn_norm.variance_epsilon,
|
||||
capture_aux=capture_previous_aux,
|
||||
stream=mhc_stream,
|
||||
)
|
||||
if capture_previous_aux:
|
||||
previous_aux = aux
|
||||
@@ -448,6 +484,8 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
if self.use_sequence_parallel:
|
||||
x = sp_reduce_scatter(x)
|
||||
|
||||
if mhc_stream is not None:
|
||||
torch.cuda.current_stream().wait_stream(mhc_stream)
|
||||
residual, post_mix, res_mix, x, ffn_pre, _ = mhc_shifted_post_pre(
|
||||
x,
|
||||
residual,
|
||||
@@ -464,8 +502,11 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
pre_mix=attn_pre,
|
||||
norm_weight=self.ffn_norm.weight,
|
||||
norm_eps=self.ffn_norm.variance_epsilon,
|
||||
stream=mhc_stream,
|
||||
)
|
||||
x = self.ffn(x, input_ids, mega_gate_metadata)
|
||||
if mhc_stream is not None:
|
||||
torch.cuda.current_stream().wait_stream(mhc_stream)
|
||||
return x, residual, post_mix, res_mix, ffn_pre, previous_aux
|
||||
|
||||
|
||||
@@ -497,6 +538,8 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
# (compressor kv_score, indexer.weights_proj). fused_wqa_wkv stays on
|
||||
# the default stream.
|
||||
aux_stream_list = [torch.cuda.Stream() for _ in range(3)]
|
||||
# Keep mHC independent of the streams used inside attention.
|
||||
mhc_stream = torch.cuda.Stream() if supports_mhc_overlap(vllm_config) else None
|
||||
|
||||
# Reserved topk indices buffer for all Indexer layers to reuse.
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
@@ -541,6 +584,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
aux_stream_list=aux_stream_list,
|
||||
candidate_block_buffer=self.candidate_block_buffer,
|
||||
engram_layout=self.engram_layout,
|
||||
mhc_stream=mhc_stream,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
@@ -7,6 +7,7 @@ import torch
|
||||
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
mhc_fused_post_pre_delayed_tilelang,
|
||||
mhc_post_tilelang,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
@@ -106,10 +107,37 @@ def mhc_shifted_post_pre(
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 1e-6,
|
||||
capture_aux: bool = False,
|
||||
*,
|
||||
stream: torch.cuda.Stream | None = None,
|
||||
) -> tuple[
|
||||
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
|
||||
]:
|
||||
"""Use Mega-mHC for carried mixing, retaining TileLang for aux capture."""
|
||||
"""Dispatch shifted post/pre to overlap, Mega-mHC, or fused TileLang.
|
||||
|
||||
When stream is supplied, join it before consuming the returned coefficients.
|
||||
"""
|
||||
if stream is not None:
|
||||
from .mhc import mhc_pre_delayed_overlap
|
||||
|
||||
residual = mhc_post_tilelang(x, residual, post_layer_mix, comb_res_mix)
|
||||
aux = residual.mean(dim=1) if capture_aux else x.new_empty(0, x.shape[1])
|
||||
pre_outputs = mhc_pre_delayed_overlap(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix=pre_mix,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
stream=stream,
|
||||
)
|
||||
return residual, *pre_outputs, aux
|
||||
|
||||
if (
|
||||
pre_mix is not None
|
||||
and norm_weight is not None
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Split shifted mHC pre so coefficient generation can overlap the sublayer."""
|
||||
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
# GB200 TP4/FlashInfer improves through 16 tokens; larger screens tie or regress.
|
||||
MHC_OVERLAP_MAX_TOKENS = 16
|
||||
|
||||
|
||||
def supports_mhc_overlap(vllm_config: VllmConfig) -> bool:
|
||||
"""Check kernel requirements and safety of sharing the coefficient stream."""
|
||||
config = vllm_config.model_config.hf_config
|
||||
# DeepGEMM's prenorm kernel requires K % 64 == 0, N % 8 == 0, and N <= 32.
|
||||
mix_size = config.hc_mult * (config.hc_mult + 2)
|
||||
return (
|
||||
current_platform.is_device_capability_family(100)
|
||||
and is_deep_gemm_supported()
|
||||
and config.hidden_size % 64 == 0
|
||||
and 0 < mix_size <= 32
|
||||
and mix_size % 8 == 0
|
||||
and not vllm_config.parallel_config.use_ubatching
|
||||
)
|
||||
|
||||
|
||||
def mhc_pre_delayed_overlap(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
x: torch.Tensor | None = None,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 1e-6,
|
||||
*,
|
||||
stream: torch.cuda.Stream,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Prepare the input on the caller stream and coefficients on another stream.
|
||||
|
||||
Returns post mix, residual mix, normalized input, and next pre-mix. Only the
|
||||
input is ready on the caller stream; join stream before using coefficients.
|
||||
The caller must retain the weights and coefficient outputs until the join.
|
||||
"""
|
||||
from vllm.model_executor.kernels.mhc.warmup import (
|
||||
MHC_PRE_NORM_KERNEL,
|
||||
compute_mhc_pre_num_splits,
|
||||
)
|
||||
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
|
||||
|
||||
n, hc, hidden = residual.shape
|
||||
assert residual.is_contiguous() and residual.dtype == torch.bfloat16
|
||||
assert norm_weight is not None
|
||||
if x is None:
|
||||
x = residual.view(n, hc * hidden)
|
||||
assert x.is_contiguous()
|
||||
post = torch.empty((n, hc), device=residual.device, dtype=torch.float32)
|
||||
comb = torch.empty((n, hc * hc), device=residual.device, dtype=torch.float32)
|
||||
next_pre = torch.empty_like(post)
|
||||
layer_input = torch.empty((n, hidden), device=residual.device, dtype=torch.bfloat16)
|
||||
outputs = post.unsqueeze(-1), comb.view(n, hc, hc), layer_input, next_pre
|
||||
if n == 0:
|
||||
return outputs
|
||||
splits = compute_mhc_pre_num_splits(x.shape[1], n)
|
||||
mix = torch.empty(
|
||||
(splits, n, hc * (hc + 2)), device=residual.device, dtype=torch.float32
|
||||
)
|
||||
sqr = torch.empty((splits, n), device=residual.device, dtype=torch.float32)
|
||||
epilogue = partial(
|
||||
MHC_PRE_NORM_KERNEL,
|
||||
mix,
|
||||
sqr,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual,
|
||||
post,
|
||||
comb,
|
||||
layer_input,
|
||||
norm_weight,
|
||||
pre_mix if pre_mix is not None else post,
|
||||
next_pre,
|
||||
layer_input, # Unused aux output in split modes.
|
||||
hidden_size=hidden,
|
||||
rms_eps=rms_eps,
|
||||
hc_pre_eps=hc_pre_eps,
|
||||
hc_sinkhorn_eps=hc_sinkhorn_eps,
|
||||
hc_post_mult_value=hc_post_mult_value,
|
||||
sinkhorn_repeat=sinkhorn_repeat,
|
||||
norm_eps=norm_eps,
|
||||
hc_mult=hc,
|
||||
use_pre_mix_in=pre_mix is not None,
|
||||
save_pre_mix=True,
|
||||
rms_numel=x.shape[1],
|
||||
)
|
||||
main = torch.cuda.current_stream()
|
||||
# Prioritize input readiness for small batches before releasing statistics.
|
||||
if n <= 8:
|
||||
epilogue(split_mode="input")
|
||||
stream.wait_stream(main)
|
||||
with torch.cuda.stream(stream):
|
||||
tf32_hc_prenorm_gemm(x, fn, mix, sqr, splits)
|
||||
epilogue(split_mode="stats")
|
||||
for tensor in (x, mix, sqr):
|
||||
tensor.record_stream(stream)
|
||||
if n > 8:
|
||||
epilogue(split_mode="input")
|
||||
return outputs
|
||||
Reference in New Issue
Block a user