[Perf][DSV4.1] Overlap mHC coefficients for small TP batches (#57603)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -266,6 +266,77 @@ def test_deepseek_v41_mhc_pre_delayed(
|
||||
torch.testing.assert_close(actual[2], expected[2], atol=1.6e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not HAS_TILELANG_MHC or not current_platform.is_device_capability_family(100),
|
||||
reason="SM100 TileLang mHC required",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [0, 1, 8, 9, 32, 128, 256])
|
||||
@pytest.mark.parametrize("entry", ["broadcast", "identity", "carried"])
|
||||
@pytest.mark.parametrize("hidden_size,hc_mult", [(4096, 2), (5120, 4), (7168, 4)])
|
||||
def test_deepseek_v41_mhc_overlap_preserves_outputs_on_replay(
|
||||
num_tokens, entry, hidden_size, hc_mult
|
||||
):
|
||||
"""The input is usable before joining; coefficients stay exact on graph replay."""
|
||||
from vllm.models.deepseek_v41.nvidia.ops.mhc import mhc_pre_delayed_overlap
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
if not is_deep_gemm_supported():
|
||||
pytest.skip("DeepGEMM required")
|
||||
set_random_seed(42)
|
||||
mix_size = hc_mult * (hc_mult + 2)
|
||||
residual = torch.randn(
|
||||
num_tokens, hc_mult, hidden_size, dtype=torch.bfloat16, device=DEVICE
|
||||
)
|
||||
fn = torch.randn(mix_size, hc_mult, hidden_size, device=DEVICE) * 0.02
|
||||
x = None
|
||||
pre_mix = None
|
||||
if entry == "broadcast":
|
||||
x = residual[:, 0].contiguous()
|
||||
residual = x.unsqueeze(1).expand(-1, hc_mult, -1).contiguous()
|
||||
fn = fn.sum(1)
|
||||
else:
|
||||
fn = fn.flatten(1)
|
||||
if entry == "carried":
|
||||
pre_mix = torch.rand(num_tokens, hc_mult, device=DEVICE)
|
||||
scale = torch.tensor([0.5, 0.25, 1.0], device=DEVICE)
|
||||
base = torch.randn(mix_size, device=DEVICE)
|
||||
weight = torch.empty(hidden_size, dtype=torch.bfloat16, device=DEVICE).uniform_(
|
||||
0.5, 1.5
|
||||
)
|
||||
args = (residual, fn, scale, base, 1e-20, 1e-6, 1e-6, 2.0, 20)
|
||||
kwargs = dict(pre_mix=pre_mix, x=x, norm_weight=weight, norm_eps=1e-20)
|
||||
side = torch.cuda.Stream()
|
||||
|
||||
def run():
|
||||
outputs = mhc_pre_delayed_overlap(*args, **kwargs, stream=side)
|
||||
consumed_input = outputs[2].clone()
|
||||
torch.cuda.current_stream().wait_stream(side)
|
||||
return (*outputs, consumed_input)
|
||||
|
||||
def check(actual):
|
||||
expected = mhc_pre_delayed_tilelang(*args, **kwargs)
|
||||
for a, b in zip(actual[:4], expected, strict=True):
|
||||
torch.testing.assert_close(a, b, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual[4], expected[2], atol=0, rtol=0)
|
||||
|
||||
check(run())
|
||||
if num_tokens == 0:
|
||||
return
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
captured = run()
|
||||
for _ in range(5):
|
||||
if x is not None:
|
||||
x.normal_()
|
||||
residual.copy_(x.unsqueeze(1).expand_as(residual))
|
||||
else:
|
||||
residual.normal_()
|
||||
if pre_mix is not None:
|
||||
pre_mix.uniform_()
|
||||
graph.replay()
|
||||
check(captured)
|
||||
|
||||
|
||||
def mhc_fused_post_pre_delayed_ref(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
@@ -433,10 +504,16 @@ def test_mhc_fused_post_pre_delayed_custom_op_supports_compile(carried, capture_
|
||||
|
||||
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
|
||||
@pytest.mark.parametrize("entry", ["broadcast", "pipeline", "residual", "engram"])
|
||||
@pytest.mark.parametrize(
|
||||
"mhc_mode", ["disabled", "overlap", "piecewise", "large_batch"]
|
||||
)
|
||||
def test_deepseek_v41_decoder_mixes_match_torch(
|
||||
entry, monkeypatch, default_vllm_config
|
||||
entry, mhc_mode, monkeypatch, default_vllm_config
|
||||
):
|
||||
"""Preserve carried pre-mixes and Engram ordering at every decoder entry."""
|
||||
from vllm.models.deepseek_v41.nvidia.ops.mhc import MHC_OVERLAP_MAX_TOKENS
|
||||
|
||||
num_tokens = MHC_OVERLAP_MAX_TOKENS + 1 if mhc_mode == "large_batch" else 3
|
||||
set_random_seed(0)
|
||||
decoder = DeepseekV41DecoderLayer.__new__(DeepseekV41DecoderLayer)
|
||||
nn.Module.__init__(decoder)
|
||||
@@ -446,6 +523,29 @@ def test_deepseek_v41_decoder_mixes_match_torch(
|
||||
decoder.rms_norm_eps = 1e-20
|
||||
decoder.hc_post_alpha = 2.0
|
||||
decoder.use_sequence_parallel = False
|
||||
decoder.mhc_stream = None
|
||||
if mhc_mode != "disabled":
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
if (
|
||||
not current_platform.is_device_capability_family(100)
|
||||
or not is_deep_gemm_supported()
|
||||
):
|
||||
pytest.skip("SM100 DeepGEMM required for overlap")
|
||||
decoder.mhc_stream = torch.cuda.Stream()
|
||||
if mhc_mode in ("piecewise", "large_batch"):
|
||||
|
||||
def unexpected_overlap(*args, **kwargs):
|
||||
pytest.fail("unsupported execution must retain native mHC")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v41.nvidia.model.mhc_pre_delayed_overlap",
|
||||
unexpected_overlap,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v41.nvidia.ops.mhc.mhc_pre_delayed_overlap",
|
||||
unexpected_overlap,
|
||||
)
|
||||
decoder.engram = None
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
|
||||
@@ -461,18 +561,18 @@ def test_deepseek_v41_decoder_mixes_match_torch(
|
||||
decoder.hc_attn_scale = decoder.hc_ffn_scale = torch.ones(3)
|
||||
decoder.hc_attn_base = torch.randn(24)
|
||||
decoder.hc_ffn_base = torch.randn(24)
|
||||
x = torch.randn(3, 5120, dtype=torch.bfloat16)
|
||||
positions = torch.arange(3)
|
||||
x = torch.randn(num_tokens, 5120, dtype=torch.bfloat16)
|
||||
positions = torch.arange(num_tokens)
|
||||
kwargs = {}
|
||||
if entry != "broadcast":
|
||||
kwargs["pre_mix"] = torch.rand(3, 4)
|
||||
kwargs["pre_mix"] = torch.rand(num_tokens, 4)
|
||||
if entry == "pipeline":
|
||||
x = torch.randn(3, 4, 5120, dtype=torch.bfloat16)
|
||||
x = torch.randn(num_tokens, 4, 5120, dtype=torch.bfloat16)
|
||||
if entry in ("residual", "engram"):
|
||||
kwargs.update(
|
||||
residual=torch.randn(3, 4, 5120, dtype=torch.bfloat16),
|
||||
post_mix=torch.rand(3, 4, 1),
|
||||
res_mix=torch.rand(3, 4, 4),
|
||||
residual=torch.randn(num_tokens, 4, 5120, dtype=torch.bfloat16),
|
||||
post_mix=torch.rand(num_tokens, 4, 1),
|
||||
res_mix=torch.rand(num_tokens, 4, 4),
|
||||
)
|
||||
if entry == "engram":
|
||||
|
||||
@@ -483,15 +583,22 @@ def test_deepseek_v41_decoder_mixes_match_torch(
|
||||
return residual + 0.125
|
||||
|
||||
decoder.engram = FakeEngram()
|
||||
kwargs["engram_hashes"] = torch.zeros(3, 1, 1, dtype=torch.int32)
|
||||
kwargs["engram_hashes"] = torch.zeros(num_tokens, 1, 1, dtype=torch.int32)
|
||||
|
||||
actual = decoder(x, positions, None, **kwargs)
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.forward_context import set_forward_context
|
||||
|
||||
mode = CUDAGraphMode.PIECEWISE if mhc_mode == "piecewise" else CUDAGraphMode.NONE
|
||||
with set_forward_context(None, default_vllm_config, cudagraph_runtime_mode=mode):
|
||||
actual = decoder(x, positions, None, **kwargs)
|
||||
|
||||
def reference(*args, norm_weight, norm_eps, **kwargs):
|
||||
post, res, collapsed, pre = mhc_pre_delayed_torch(*args, **kwargs)
|
||||
return post, res, decoder.attn_norm(collapsed), pre
|
||||
|
||||
def fused_reference(x, residual, post_mix, res_mix, *args, capture_aux=False, **kw):
|
||||
def fused_reference(
|
||||
x, residual, post_mix, res_mix, *args, capture_aux=False, stream=None, **kw
|
||||
):
|
||||
residual = mhc_post_torch(x, residual, post_mix, res_mix)
|
||||
aux = residual.mean(dim=1) if capture_aux else residual.new_empty(0)
|
||||
return residual, *reference(residual, *args, **kw), aux
|
||||
@@ -500,6 +607,7 @@ def test_deepseek_v41_decoder_mixes_match_torch(
|
||||
"vllm.models.deepseek_v41.nvidia.model.mhc_pre_delayed_tilelang",
|
||||
reference,
|
||||
)
|
||||
decoder.mhc_stream = None
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.deepseek_v41.nvidia.model.mhc_shifted_post_pre",
|
||||
fused_reference,
|
||||
@@ -530,6 +638,7 @@ def test_deepseek_v41_capture_previous_aux(entry, monkeypatch, default_vllm_conf
|
||||
decoder.rms_norm_eps = 1e-6
|
||||
decoder.hc_post_alpha = 2.0
|
||||
decoder.use_sequence_parallel = False
|
||||
decoder.mhc_stream = None
|
||||
decoder.engram = None
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
|
||||
@@ -772,7 +881,8 @@ def hc_head_ref(
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 8, 128])
|
||||
@pytest.mark.parametrize("hidden_size", [4096, 7168])
|
||||
@pytest.mark.parametrize("hc_mult", [4])
|
||||
def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult):
|
||||
@pytest.mark.parametrize("fused_norm", [False, True])
|
||||
def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult, fused_norm):
|
||||
torch.set_default_device(DEVICE)
|
||||
set_random_seed(0)
|
||||
|
||||
@@ -802,6 +912,12 @@ def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult):
|
||||
hc_post_alpha,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
norm_kwargs = {}
|
||||
if fused_norm:
|
||||
# Non-shifted mHC must still use its freshly computed pre-mix.
|
||||
weight = torch.ones(hidden_size, dtype=torch.bfloat16)
|
||||
norm_kwargs = dict(norm_weight=weight, norm_eps=rms_eps)
|
||||
ref = (*ref[:2], F.rms_norm(ref[2], (hidden_size,), weight, rms_eps))
|
||||
out = torch.ops.vllm.mhc_pre_tilelang(
|
||||
residual,
|
||||
fn,
|
||||
@@ -812,6 +928,7 @@ def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult):
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_alpha,
|
||||
sinkhorn_repeat,
|
||||
**norm_kwargs,
|
||||
)
|
||||
|
||||
for actual, expected in zip(out, ref, strict=True):
|
||||
|
||||
@@ -305,8 +305,12 @@ def mhc_pre_big_fuse_with_norm_tilelang(
|
||||
use_pre_mix_in: bool = False,
|
||||
save_pre_mix: bool = False,
|
||||
rms_numel: int = 0,
|
||||
split_mode: str = "fused",
|
||||
write_aux: bool = False,
|
||||
):
|
||||
# Split modes require shifted mHC: input collapse uses a carried pre-mix.
|
||||
assert split_mode in ("fused", "stats", "input")
|
||||
assert split_mode == "fused" or save_pre_mix
|
||||
num_tokens = T.dynamic("num_tokens")
|
||||
hc_mult3 = hc_mult * (2 + hc_mult)
|
||||
if gemm_last_dim < 0:
|
||||
@@ -330,26 +334,26 @@ def mhc_pre_big_fuse_with_norm_tilelang(
|
||||
aux_out: T.Tensor[[num_tokens, hidden_size], T.bfloat16] # type: ignore[no-redef, valid-type]
|
||||
|
||||
with T.Kernel(num_tokens, threads=96) as i:
|
||||
rms = T.alloc_fragment(1, T.float32)
|
||||
mixes = T.alloc_fragment(hc_mult3, T.float32)
|
||||
T.clear(mixes)
|
||||
rms[0] = 0
|
||||
|
||||
if ENABLE_PDL:
|
||||
T.pdl_sync()
|
||||
|
||||
for i_split in T.serial(n_splits):
|
||||
rms[0] += gemm_out_sqrsum[i_split, i]
|
||||
rms[0] = T.rsqrt(rms[0] / rms_numel + rms_eps)
|
||||
for j in T.Parallel(hc_mult3):
|
||||
mixes[j] = 0
|
||||
for i_split in T.serial(n_splits):
|
||||
mixes[j] += gemm_out_mul[i_split, i, j]
|
||||
mixes[j] *= rms[0]
|
||||
mixes_shared = T.alloc_shared(hc_mult3, T.float32)
|
||||
T.copy(mixes, mixes_shared)
|
||||
if split_mode != "input":
|
||||
rms = T.alloc_fragment(1, T.float32)
|
||||
mixes = T.alloc_fragment(hc_mult3, T.float32)
|
||||
T.clear(mixes)
|
||||
rms[0] = 0
|
||||
|
||||
if T.get_thread_binding() < 32:
|
||||
for i_split in T.serial(n_splits):
|
||||
rms[0] += gemm_out_sqrsum[i_split, i]
|
||||
rms[0] = T.rsqrt(rms[0] / rms_numel + rms_eps)
|
||||
for j in T.Parallel(hc_mult3):
|
||||
mixes[j] = 0
|
||||
for i_split in T.serial(n_splits):
|
||||
mixes[j] += gemm_out_mul[i_split, i, j]
|
||||
mixes[j] *= rms[0]
|
||||
T.copy(mixes, mixes_shared)
|
||||
|
||||
if split_mode != "input" and T.get_thread_binding() < 32:
|
||||
cm = T.alloc_fragment((hc_mult, hc_mult), T.float32)
|
||||
for j in T.Parallel(hc_mult):
|
||||
if save_pre_mix:
|
||||
@@ -395,7 +399,7 @@ def mhc_pre_big_fuse_with_norm_tilelang(
|
||||
|
||||
for j, k in T.Parallel(hc_mult, hc_mult):
|
||||
comb_mix[i, j * hc_mult + k] = cm[j, k]
|
||||
else:
|
||||
elif split_mode != "stats" and T.get_thread_binding() >= 32:
|
||||
pre_mix_shared = T.alloc_shared(hc_mult, T.float32)
|
||||
for j in T.Parallel(hc_mult):
|
||||
if use_pre_mix_in:
|
||||
|
||||
@@ -36,6 +36,7 @@ class MHCPreNormKernel(VllmJitKernel["MHCPreNormKernel.CompileKey"]):
|
||||
use_pre_mix_in: bool
|
||||
rms_numel: int
|
||||
save_pre_mix: bool = True
|
||||
split_mode: str = "fused"
|
||||
write_aux: bool = False
|
||||
|
||||
kernel: Any = staticmethod(mhc_pre_big_fuse_with_norm_tilelang)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import typing
|
||||
from collections.abc import Callable, Iterable
|
||||
from functools import partial
|
||||
from itertools import islice
|
||||
|
||||
import regex as re
|
||||
@@ -17,7 +18,11 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.forward_context import (
|
||||
get_forward_context,
|
||||
in_piecewise_cudagraph,
|
||||
is_forward_context_available,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
mhc_post_tilelang,
|
||||
@@ -84,6 +89,11 @@ from ..common.engram import EngramLayout, NgramHashState
|
||||
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
|
||||
from .engram import Engram, gather_engram_hashes
|
||||
from .ops.mega_mhc import mhc_shifted_post_pre
|
||||
from .ops.mhc import (
|
||||
MHC_OVERLAP_MAX_TOKENS,
|
||||
mhc_pre_delayed_overlap,
|
||||
supports_mhc_overlap,
|
||||
)
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
@@ -188,11 +198,13 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
aux_stream_list: list[torch.cuda.Stream] | None = None,
|
||||
candidate_block_buffer: torch.Tensor | None = None,
|
||||
engram_layout: EngramLayout | None = None,
|
||||
mhc_stream: torch.cuda.Stream | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.hidden_size = config.hidden_size
|
||||
self.mhc_stream = mhc_stream
|
||||
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
|
||||
|
||||
self.engram: Engram | None = None
|
||||
@@ -269,19 +281,30 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
variants.append(
|
||||
{**hc_stream, "use_pre_mix_in": True, "write_aux": True}
|
||||
)
|
||||
# Shifted mHC splits the epilogue in two: the input collapse on the
|
||||
# caller stream and coefficient generation on a side stream.
|
||||
split_modes = (
|
||||
("fused", "stats", "input") if mhc_stream is not None else ("fused",)
|
||||
)
|
||||
for variant in variants:
|
||||
MHC_PRE_NORM_KERNEL.register_warmup(
|
||||
max_tokens=max_tokens,
|
||||
hidden_size=self.hidden_size,
|
||||
rms_eps=self.rms_norm_eps,
|
||||
hc_pre_eps=self.hc_eps,
|
||||
hc_sinkhorn_eps=self.hc_eps,
|
||||
hc_post_mult_value=self.hc_post_alpha,
|
||||
sinkhorn_repeat=self.hc_sinkhorn_iters,
|
||||
norm_eps=self.rms_norm_eps,
|
||||
hc_mult=self.hc_mult,
|
||||
**variant,
|
||||
)
|
||||
for split_mode in split_modes:
|
||||
MHC_PRE_NORM_KERNEL.register_warmup(
|
||||
split_mode=split_mode,
|
||||
max_tokens=(
|
||||
max_tokens
|
||||
if split_mode == "fused"
|
||||
else min(max_tokens, MHC_OVERLAP_MAX_TOKENS)
|
||||
),
|
||||
hidden_size=self.hidden_size,
|
||||
rms_eps=self.rms_norm_eps,
|
||||
hc_pre_eps=self.hc_eps,
|
||||
hc_sinkhorn_eps=self.hc_eps,
|
||||
hc_post_mult_value=self.hc_post_alpha,
|
||||
sinkhorn_repeat=self.hc_sinkhorn_iters,
|
||||
norm_eps=self.rms_norm_eps,
|
||||
hc_mult=self.hc_mult,
|
||||
**variant,
|
||||
)
|
||||
mix_hc = (2 + self.hc_mult) * self.hc_mult
|
||||
hc_dim = self.hc_mult * self.hidden_size
|
||||
self.hc_attn_fn = nn.Parameter(
|
||||
@@ -351,6 +374,18 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
torch.Tensor | None,
|
||||
]:
|
||||
previous_aux: torch.Tensor | None = None
|
||||
mhc_stream = self.mhc_stream
|
||||
if mhc_stream is not None and (
|
||||
in_piecewise_cudagraph()
|
||||
or not 0 < positions.shape[0] <= MHC_OVERLAP_MAX_TOKENS
|
||||
):
|
||||
# A side stream cannot remain unjoined across breakable graph segments.
|
||||
mhc_stream = None
|
||||
mhc_pre = (
|
||||
partial(mhc_pre_delayed_overlap, stream=mhc_stream)
|
||||
if mhc_stream is not None
|
||||
else mhc_pre_delayed_tilelang
|
||||
)
|
||||
# The reference collapses each sublayer's input with the *previous*
|
||||
# sublayer's pre-mix: attention uses the pre-mix carried in (identity
|
||||
# for the first layer), the FFN uses this layer's attention pre-mix.
|
||||
@@ -360,7 +395,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
# copies and the identity pre-mix selects copy 0.
|
||||
assert self.hc_attn_fn_broadcast is not None
|
||||
residual = x.unsqueeze(1).expand(-1, self.hc_mult, -1).contiguous()
|
||||
post_mix, res_mix, x, attn_pre = mhc_pre_delayed_tilelang(
|
||||
post_mix, res_mix, x, attn_pre = mhc_pre(
|
||||
residual,
|
||||
self.hc_attn_fn_broadcast,
|
||||
self.hc_attn_scale,
|
||||
@@ -376,7 +411,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
)
|
||||
else:
|
||||
residual = x
|
||||
post_mix, res_mix, x, attn_pre = mhc_pre_delayed_tilelang(
|
||||
post_mix, res_mix, x, attn_pre = mhc_pre(
|
||||
residual,
|
||||
self.hc_attn_fn,
|
||||
self.hc_attn_scale,
|
||||
@@ -403,7 +438,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
engram_hashes[:, self.engram.layer_hash_index],
|
||||
engram_mask,
|
||||
)
|
||||
post_mix, res_mix, x, attn_pre = mhc_pre_delayed_tilelang(
|
||||
post_mix, res_mix, x, attn_pre = mhc_pre(
|
||||
residual,
|
||||
self.hc_attn_fn,
|
||||
self.hc_attn_scale,
|
||||
@@ -437,6 +472,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
norm_weight=self.attn_norm.weight,
|
||||
norm_eps=self.attn_norm.variance_epsilon,
|
||||
capture_aux=capture_previous_aux,
|
||||
stream=mhc_stream,
|
||||
)
|
||||
if capture_previous_aux:
|
||||
previous_aux = aux
|
||||
@@ -448,6 +484,8 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
if self.use_sequence_parallel:
|
||||
x = sp_reduce_scatter(x)
|
||||
|
||||
if mhc_stream is not None:
|
||||
torch.cuda.current_stream().wait_stream(mhc_stream)
|
||||
residual, post_mix, res_mix, x, ffn_pre, _ = mhc_shifted_post_pre(
|
||||
x,
|
||||
residual,
|
||||
@@ -464,8 +502,11 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
pre_mix=attn_pre,
|
||||
norm_weight=self.ffn_norm.weight,
|
||||
norm_eps=self.ffn_norm.variance_epsilon,
|
||||
stream=mhc_stream,
|
||||
)
|
||||
x = self.ffn(x, input_ids, mega_gate_metadata)
|
||||
if mhc_stream is not None:
|
||||
torch.cuda.current_stream().wait_stream(mhc_stream)
|
||||
return x, residual, post_mix, res_mix, ffn_pre, previous_aux
|
||||
|
||||
|
||||
@@ -497,6 +538,8 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
# (compressor kv_score, indexer.weights_proj). fused_wqa_wkv stays on
|
||||
# the default stream.
|
||||
aux_stream_list = [torch.cuda.Stream() for _ in range(3)]
|
||||
# Keep mHC independent of the streams used inside attention.
|
||||
mhc_stream = torch.cuda.Stream() if supports_mhc_overlap(vllm_config) else None
|
||||
|
||||
# Reserved topk indices buffer for all Indexer layers to reuse.
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
@@ -541,6 +584,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
aux_stream_list=aux_stream_list,
|
||||
candidate_block_buffer=self.candidate_block_buffer,
|
||||
engram_layout=self.engram_layout,
|
||||
mhc_stream=mhc_stream,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
@@ -7,6 +7,7 @@ import torch
|
||||
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
mhc_fused_post_pre_delayed_tilelang,
|
||||
mhc_post_tilelang,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
@@ -106,10 +107,37 @@ def mhc_shifted_post_pre(
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 1e-6,
|
||||
capture_aux: bool = False,
|
||||
*,
|
||||
stream: torch.cuda.Stream | None = None,
|
||||
) -> tuple[
|
||||
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
|
||||
]:
|
||||
"""Use Mega-mHC for carried mixing, retaining TileLang for aux capture."""
|
||||
"""Dispatch shifted post/pre to overlap, Mega-mHC, or fused TileLang.
|
||||
|
||||
When stream is supplied, join it before consuming the returned coefficients.
|
||||
"""
|
||||
if stream is not None:
|
||||
from .mhc import mhc_pre_delayed_overlap
|
||||
|
||||
residual = mhc_post_tilelang(x, residual, post_layer_mix, comb_res_mix)
|
||||
aux = residual.mean(dim=1) if capture_aux else x.new_empty(0, x.shape[1])
|
||||
pre_outputs = mhc_pre_delayed_overlap(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix=pre_mix,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
stream=stream,
|
||||
)
|
||||
return residual, *pre_outputs, aux
|
||||
|
||||
if (
|
||||
pre_mix is not None
|
||||
and norm_weight is not None
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Split shifted mHC pre so coefficient generation can overlap the sublayer."""
|
||||
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
# GB200 TP4/FlashInfer improves through 16 tokens; larger screens tie or regress.
|
||||
MHC_OVERLAP_MAX_TOKENS = 16
|
||||
|
||||
|
||||
def supports_mhc_overlap(vllm_config: VllmConfig) -> bool:
|
||||
"""Check kernel requirements and safety of sharing the coefficient stream."""
|
||||
config = vllm_config.model_config.hf_config
|
||||
# DeepGEMM's prenorm kernel requires K % 64 == 0, N % 8 == 0, and N <= 32.
|
||||
mix_size = config.hc_mult * (config.hc_mult + 2)
|
||||
return (
|
||||
current_platform.is_device_capability_family(100)
|
||||
and is_deep_gemm_supported()
|
||||
and config.hidden_size % 64 == 0
|
||||
and 0 < mix_size <= 32
|
||||
and mix_size % 8 == 0
|
||||
and not vllm_config.parallel_config.use_ubatching
|
||||
)
|
||||
|
||||
|
||||
def mhc_pre_delayed_overlap(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
x: torch.Tensor | None = None,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 1e-6,
|
||||
*,
|
||||
stream: torch.cuda.Stream,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Prepare the input on the caller stream and coefficients on another stream.
|
||||
|
||||
Returns post mix, residual mix, normalized input, and next pre-mix. Only the
|
||||
input is ready on the caller stream; join stream before using coefficients.
|
||||
The caller must retain the weights and coefficient outputs until the join.
|
||||
"""
|
||||
from vllm.model_executor.kernels.mhc.warmup import (
|
||||
MHC_PRE_NORM_KERNEL,
|
||||
compute_mhc_pre_num_splits,
|
||||
)
|
||||
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
|
||||
|
||||
n, hc, hidden = residual.shape
|
||||
assert residual.is_contiguous() and residual.dtype == torch.bfloat16
|
||||
assert norm_weight is not None
|
||||
if x is None:
|
||||
x = residual.view(n, hc * hidden)
|
||||
assert x.is_contiguous()
|
||||
post = torch.empty((n, hc), device=residual.device, dtype=torch.float32)
|
||||
comb = torch.empty((n, hc * hc), device=residual.device, dtype=torch.float32)
|
||||
next_pre = torch.empty_like(post)
|
||||
layer_input = torch.empty((n, hidden), device=residual.device, dtype=torch.bfloat16)
|
||||
outputs = post.unsqueeze(-1), comb.view(n, hc, hc), layer_input, next_pre
|
||||
if n == 0:
|
||||
return outputs
|
||||
splits = compute_mhc_pre_num_splits(x.shape[1], n)
|
||||
mix = torch.empty(
|
||||
(splits, n, hc * (hc + 2)), device=residual.device, dtype=torch.float32
|
||||
)
|
||||
sqr = torch.empty((splits, n), device=residual.device, dtype=torch.float32)
|
||||
epilogue = partial(
|
||||
MHC_PRE_NORM_KERNEL,
|
||||
mix,
|
||||
sqr,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual,
|
||||
post,
|
||||
comb,
|
||||
layer_input,
|
||||
norm_weight,
|
||||
pre_mix if pre_mix is not None else post,
|
||||
next_pre,
|
||||
layer_input, # Unused aux output in split modes.
|
||||
hidden_size=hidden,
|
||||
rms_eps=rms_eps,
|
||||
hc_pre_eps=hc_pre_eps,
|
||||
hc_sinkhorn_eps=hc_sinkhorn_eps,
|
||||
hc_post_mult_value=hc_post_mult_value,
|
||||
sinkhorn_repeat=sinkhorn_repeat,
|
||||
norm_eps=norm_eps,
|
||||
hc_mult=hc,
|
||||
use_pre_mix_in=pre_mix is not None,
|
||||
save_pre_mix=True,
|
||||
rms_numel=x.shape[1],
|
||||
)
|
||||
main = torch.cuda.current_stream()
|
||||
# Prioritize input readiness for small batches before releasing statistics.
|
||||
if n <= 8:
|
||||
epilogue(split_mode="input")
|
||||
stream.wait_stream(main)
|
||||
with torch.cuda.stream(stream):
|
||||
tf32_hc_prenorm_gemm(x, fn, mix, sqr, splits)
|
||||
epilogue(split_mode="stats")
|
||||
for tensor in (x, mix, sqr):
|
||||
tensor.record_stream(stream)
|
||||
if n > 8:
|
||||
epilogue(split_mode="input")
|
||||
return outputs
|
||||
Reference in New Issue
Block a user