[ROCm][DSV4.1][Perf] Use AITER mHC for the delayed pre block (#56503)

Signed-off-by: Fangzhou Ai <fangzhou@semianalysis.ai>
Co-authored-by: Fangzhou Ai <fangzhou@semianalysis.ai>
This commit is contained in:
Fangzhou Ai
2026-09-12 00:35:48 +00:00
committed by GitHub
co-authored by Fangzhou Ai
parent 9d88ceb026
commit 46d2b23ac5
7 changed files with 620 additions and 19 deletions
+95
View File
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.mhc import (
HAS_AITER_MHC_PRE_NORM,
HAS_TILELANG_MHC,
MHCFusedPostPreOp,
MHCPreDelayedOp,
MHCPreOp,
)
from vllm.models.deepseek_v4.nvidia.model import (
@@ -697,6 +698,100 @@ def test_mhc_fused_rocm_fallback_applies_norm(monkeypatch):
torch.testing.assert_close(out[3], expected_layer_input)
@pytest.mark.skipif(
not (current_platform.is_rocm() and HAS_AITER_MHC),
reason="AITER mHC required",
)
@pytest.mark.parametrize("num_tokens", [1, 2, 7, 128, 1024])
@pytest.mark.parametrize("carried", [False, True])
def test_mhc_pre_delayed_rocm_aiter(num_tokens, carried):
"""AITER must reproduce the delayed reference on both seam variants.
``num_tokens`` spans the split-k choices AITER makes for the projection,
since the pre-mix is recovered from that unreduced output.
"""
set_random_seed(0)
hc_mult, hidden_size = 4, 5120
residual, fn, hc_scale, hc_base, _ = _rocm_mhc_inputs(
num_tokens=num_tokens, hidden_size=hidden_size, hc_mult=hc_mult
)
pre_mix = (
torch.rand(num_tokens, hc_mult, dtype=torch.float32, device=DEVICE) + 0.5
if carried
else None
)
rms_eps = hc_pre_eps = hc_sinkhorn_eps = 1e-6
args = (
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
1.0,
20,
)
expected = mhc_pre_delayed_torch(*args, pre_mix=pre_mix)
actual = object.__new__(MHCPreDelayedOp).forward_hip(*args, pre_mix=pre_mix)
for i in (0, 1, 3):
torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3)
# The collapse is the same FP32 multiply-and-sum in both paths.
torch.testing.assert_close(actual[2], expected[2], atol=0, rtol=0)
@pytest.mark.skipif(
not (current_platform.is_rocm() and HAS_AITER_MHC),
reason="AITER mHC required",
)
def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch):
"""The broadcast seam and a fused norm must not take the AITER path.
Neither is expressible with AITER's pre kernels: the broadcast projects a
narrower ``x``, and ``mhc_pre_gemm_sqrsum`` folds no RMSNorm. What is
asserted here is the routing decision, which is what this gate owns; the
numerics of whichever fallback it lands on are covered by
``test_deepseek_v41_mhc_pre_delayed``.
"""
set_random_seed(0)
hc_mult, hidden_size = 4, 5120
residual, fn, hc_scale, hc_base, norm_weight = _rocm_mhc_inputs(
num_tokens=4, hidden_size=hidden_size, hc_mult=hc_mult
)
args = (residual, fn, hc_scale, hc_base, 1e-6, 1e-6, 1e-6, 1.0, 20)
op = object.__new__(MHCPreDelayedOp)
aiter_op = torch.ops.vllm.mhc_pre_delayed_aiter
took_aiter = False
def spy(*spy_args, **spy_kwargs):
nonlocal took_aiter
took_aiter = True
return aiter_op(*spy_args, **spy_kwargs)
monkeypatch.setattr(torch.ops.vllm, "mhc_pre_delayed_aiter", spy)
x = residual[:, 0].contiguous()
broadcast_fn = fn.view(-1, hc_mult, hidden_size).sum(1)
broadcast_residual = x.unsqueeze(1).expand(-1, hc_mult, -1).contiguous()
expected = mhc_pre_delayed_torch(broadcast_residual, broadcast_fn, *args[2:], x=x)
actual = op.forward_hip(broadcast_residual, broadcast_fn, *args[2:], x=x)
assert not took_aiter, "the broadcast seam must not reach AITER"
for i in range(4):
torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3)
op.forward_hip(*args, norm_weight=norm_weight, norm_eps=1e-6)
assert not took_aiter, "a fused norm must not reach AITER"
# Positive control: the same inputs without a norm do take the AITER path,
# so the two declines above are the gate discriminating rather than the
# path being unavailable in this environment.
op.forward_hip(*args)
assert took_aiter
@pytest.mark.skipif(
not (current_platform.is_rocm() and HAS_AITER_MHC and HAS_AITER_MHC_PRE_NORM),
reason="AITER mHC with fused RMSNorm required",
+131
View File
@@ -3599,6 +3599,137 @@ class rocm_aiter_ops:
layer_input.view(*outer_shape, hidden_size),
)
@staticmethod
def mhc_pre_delayed(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""mHC pre using the pre-mix carried from the previous sublayer.
Same gates as :meth:`mhc_pre`, but the stream collapse applies the
caller's ``pre_mix`` instead of the one computed here, and the one
computed here is returned for the next sublayer seam. AITER has no
single kernel for that shape, so drive its two stages directly:
``mhc_pre_big_fuse`` still produces the post and comb gates (including
every sinkhorn iteration), while the pre gate is recovered from the
same split-k GEMM output and the collapse is done by the Triton
kernel. ``mhc_pre_big_fuse`` also writes a collapse we do not use;
that redundant store is the price of not having a native delayed
kernel, and is small next to the ~140 launches it replaces.
Returns:
post_mix: shape (..., hc_mult, 1), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
next_pre_mix: shape (..., hc_mult), dtype torch.float32
"""
from aiter.ops.mhc import (
get_mhc_pre_splitk,
mhc_pre_big_fuse,
mhc_pre_gemm_sqrsum,
)
assert residual.dtype == torch.bfloat16
assert fn.dtype == torch.float32
assert hc_scale.dtype == torch.float32
assert hc_base.dtype == torch.float32
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
hc_mult3 = hc_mult * 2 + hc_mult * hc_mult
hc_hidden_size = hc_mult * hidden_size
assert fn.shape == (hc_mult3, hc_hidden_size)
assert hc_scale.shape == (3,)
assert hc_base.shape == (hc_mult3,)
outer_shape = residual.shape[:-2]
residual_flat = residual.view(-1, hc_mult, hidden_size)
num_tokens = residual_flat.shape[0]
device = residual_flat.device
if num_tokens == 0:
return (
torch.empty(0, hc_mult, 1, dtype=torch.float32, device=device),
torch.empty(0, hc_mult, hc_mult, dtype=torch.float32, device=device),
torch.empty(0, hidden_size, dtype=torch.bfloat16, device=device),
torch.empty(0, hc_mult, dtype=torch.float32, device=device),
)
pre_mix_flat = None if pre_mix is None else pre_mix.view(-1, hc_mult)
# AITER's Python wrappers allocate without explicit device arguments.
with torch.device(device):
splitk, tile_k = get_mhc_pre_splitk(num_tokens, hc_hidden_size)
# AITER pads the GEMM output to a multiple of 32 columns.
gemm_pad = torch.empty(
splitk,
num_tokens,
(hc_mult3 + 31) // 32 * 32,
dtype=torch.float32,
device=device,
)
gemm_out = gemm_pad[:, :, :hc_mult3]
sqrsum = torch.empty(splitk, num_tokens, dtype=torch.float32, device=device)
mhc_pre_gemm_sqrsum(gemm_out, sqrsum, residual_flat, fn, tile_k, 0)
post_mix = torch.empty(
num_tokens, hc_mult, 1, dtype=torch.float32, device=device
)
comb_mix = torch.empty(
num_tokens, hc_mult, hc_mult, dtype=torch.float32, device=device
)
unused_collapse = torch.empty(
num_tokens, hidden_size, dtype=torch.bfloat16, device=device
)
mhc_pre_big_fuse(
post_mix,
comb_mix,
unused_collapse,
gemm_out,
sqrsum,
hc_scale,
hc_base,
residual_flat,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
)
next_pre_mix = torch.ops.vllm.mhc_pre_mix_triton(
gemm_out,
sqrsum,
hc_scale,
hc_base,
hc_mult,
hc_hidden_size,
rms_eps,
hc_pre_eps,
)
if pre_mix_flat is None:
# Model entry selects residual stream zero.
layer_input = residual_flat[:, 0]
else:
layer_input = torch.ops.vllm.hc_collapse_triton(residual_flat, pre_mix_flat)
return (
post_mix.view(*outer_shape, hc_mult, 1),
comb_mix.view(*outer_shape, hc_mult, hc_mult),
layer_input.view(*outer_shape, hidden_size),
next_pre_mix.view(*outer_shape, hc_mult),
)
@staticmethod
def hc_head(
hs_flat: torch.Tensor,
@@ -12,6 +12,7 @@ __all__ = [
"mhc_fused_post_pre_cuda",
"hc_head_fused_kernel_cuda",
"mhc_pre_aiter",
"mhc_pre_delayed_aiter",
"mhc_post_aiter",
"mhc_fused_post_pre_aiter",
"hc_head_fused_aiter",
@@ -26,6 +27,7 @@ __all__ = [
"mhc_post_torch",
"mhc_fused_post_pre_torch",
"hc_head_fused_torch",
"mhc_pre_mix_triton",
"mhc_pre_triton",
"mhc_post_triton",
"mhc_fused_post_pre_triton",
+90
View File
@@ -104,6 +104,90 @@ def _mhc_pre_aiter_fake(
return post_mix, comb_mix, layer_input
def mhc_pre_delayed_aiter(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""mHC pre with the pre-mix carried in from the previous sublayer.
Matches ``mhc_pre_delayed_torch``: the stream collapse uses *pre_mix*
rather than the gate computed here, and that gate is returned as the
pre-mix for the next sublayer seam.
Args:
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
hc_scale: shape (3,), dtype torch.float32
hc_base: shape (hc_mult3,), dtype torch.float32
rms_eps: RMS normalization epsilon
hc_pre_eps: pre-mix epsilon
hc_sinkhorn_eps: sinkhorn epsilon
hc_post_mult_value: post-mix multiplier value
sinkhorn_repeat: number of sinkhorn iterations
pre_mix: shape (..., hc_mult) from the previous sublayer, or None at
model entry to select residual stream zero.
Returns:
post_mix: shape (..., hc_mult, 1), dtype torch.float32
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
layer_input: shape (..., hidden_size), dtype torch.bfloat16
next_pre_mix: shape (..., hc_mult), dtype torch.float32
"""
hidden_size = residual.shape[-1]
assert hidden_size % 256 == 0
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.mhc_pre_delayed(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix,
)
def _mhc_pre_delayed_aiter_fake(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
hc_mult = residual.shape[-2]
hidden_size = residual.shape[-1]
outer_shape = residual.shape[:-2]
return (
torch.empty(
*outer_shape, hc_mult, 1, dtype=torch.float32, device=residual.device
),
torch.empty(
*outer_shape, hc_mult, hc_mult, dtype=torch.float32, device=residual.device
),
torch.empty(
*outer_shape, hidden_size, dtype=torch.bfloat16, device=residual.device
),
torch.empty(*outer_shape, hc_mult, dtype=torch.float32, device=residual.device),
)
def mhc_post_aiter(
x: torch.Tensor,
residual: torch.Tensor,
@@ -226,6 +310,12 @@ direct_register_custom_op(
mutates_args=[],
fake_impl=_mhc_pre_aiter_fake,
)
direct_register_custom_op(
op_name="mhc_pre_delayed_aiter",
op_func=mhc_pre_delayed_aiter,
mutates_args=[],
fake_impl=_mhc_pre_delayed_aiter_fake,
)
direct_register_custom_op(
op_name="mhc_post_aiter",
op_func=mhc_post_aiter,
+131
View File
@@ -141,6 +141,129 @@ def _hc_collapse_triton_fake(x: Tensor, pre_mix: Tensor) -> Tensor:
return torch.empty(x.shape[0], x.shape[2], dtype=x.dtype, device=x.device)
@triton.jit
def _mhc_pre_mix_kernel(
gemm_ptr,
sqrsum_ptr,
hc_scale_ptr,
hc_base_ptr,
out_ptr,
splitk,
hc_mult: tl.constexpr,
gemm_stride_k,
gemm_stride_t,
gemm_stride_j,
sqrsum_stride_k,
sqrsum_stride_t,
out_stride_t,
out_stride_j,
inv_hc_hidden,
rms_eps,
hc_pre_eps,
SPLITK_BLOCK: tl.constexpr,
HC_BLOCK: tl.constexpr,
):
"""Recover the pre-mix gate from a split-k mHC pre GEMM output."""
token_idx = tl.program_id(0).to(tl.int64)
ks = tl.arange(0, SPLITK_BLOCK)
js = tl.arange(0, HC_BLOCK)
kmask = ks < splitk
jmask = js < hc_mult
# Only the first hc_mult GEMM columns feed the pre gate.
gemm = tl.load(
gemm_ptr
+ ks[:, None] * gemm_stride_k
+ token_idx * gemm_stride_t
+ js[None, :] * gemm_stride_j,
mask=kmask[:, None] & jmask[None, :],
other=0.0,
).to(tl.float32)
mixes = tl.sum(gemm, 0)
sqrsum = tl.load(
sqrsum_ptr + ks * sqrsum_stride_k + token_idx * sqrsum_stride_t,
mask=kmask,
other=0.0,
).to(tl.float32)
rstd = tl.rsqrt(tl.sum(sqrsum, 0) * inv_hc_hidden + rms_eps)
scale = tl.load(hc_scale_ptr).to(tl.float32)
base = tl.load(hc_base_ptr + js, mask=jmask, other=0.0).to(tl.float32)
pre = tl.sigmoid(mixes * rstd * scale + base) + hc_pre_eps
tl.store(out_ptr + token_idx * out_stride_t + js * out_stride_j, pre, mask=jmask)
def mhc_pre_mix_triton(
gemm_out: Tensor,
sqrsum: Tensor,
hc_scale: Tensor,
hc_base: Tensor,
hc_mult: int,
hc_hidden_size: int,
rms_eps: float,
hc_pre_eps: float,
) -> Tensor:
"""Pre-mix gate for the delayed mHC pre, from AITER's split-k GEMM output.
AITER's ``mhc_pre_big_fuse`` consumes the unreduced ``[splitk, tokens,
hc_mult3]`` GEMM output and the matching row square-sums, but only returns
the post and comb gates. The delayed formulation also needs the pre gate,
to carry into the next sublayer seam. It is the same slice of the same
numbers, so recover it here rather than repeating the projection.
"""
assert gemm_out.ndim == 3 and gemm_out.dtype == torch.float32
assert sqrsum.ndim == 2 and sqrsum.dtype == torch.float32
splitk, num_tokens = gemm_out.shape[0], gemm_out.shape[1]
assert sqrsum.shape == (splitk, num_tokens)
out = torch.empty(num_tokens, hc_mult, dtype=torch.float32, device=gemm_out.device)
if num_tokens == 0:
return out
_mhc_pre_mix_kernel[(num_tokens,)](
gemm_out,
sqrsum,
hc_scale,
hc_base,
out,
splitk,
hc_mult,
gemm_out.stride(0),
gemm_out.stride(1),
gemm_out.stride(2),
sqrsum.stride(0),
sqrsum.stride(1),
out.stride(0),
out.stride(1),
1.0 / hc_hidden_size,
rms_eps,
hc_pre_eps,
SPLITK_BLOCK=triton.next_power_of_2(splitk),
HC_BLOCK=triton.next_power_of_2(hc_mult),
num_warps=1,
# Match the separate FP32 multiply and add in the Torch reference.
enable_fp_fusion=False,
)
return out
def _mhc_pre_mix_triton_fake(
gemm_out: Tensor,
sqrsum: Tensor,
hc_scale: Tensor,
hc_base: Tensor,
hc_mult: int,
hc_hidden_size: int,
rms_eps: float,
hc_pre_eps: float,
) -> Tensor:
return torch.empty(
gemm_out.shape[1], hc_mult, dtype=torch.float32, device=gemm_out.device
)
def hc_head_reduce_triton_kernel(
x: torch.Tensor,
hc_fn: torch.Tensor,
@@ -216,3 +339,11 @@ direct_register_custom_op(
mutates_args=[],
fake_impl=_hc_collapse_triton_fake,
)
direct_register_custom_op(
op_name="mhc_pre_mix_triton",
op_func=mhc_pre_mix_triton,
mutates_args=[],
fake_impl=_mhc_pre_mix_triton_fake,
)
+155
View File
@@ -292,6 +292,161 @@ class MHCPreOp(CustomOp):
)
# --8<-- [start:mhc_pre_delayed]
@CustomOp.register("mhc_pre_delayed")
class MHCPreDelayedOp(CustomOp):
"""MHC pre block using the pre-mix carried from the previous sublayer.
Same gates as :class:`MHCPreOp`, but the stream collapse applies the
caller's ``pre_mix`` and this sublayer's pre-mix is returned for the next
sublayer seam. Returns post_mix, comb_mix, layer_input, next_pre_mix.
"""
# --8<-- [end:mhc_pre_delayed]
@classmethod
def enabled(cls) -> bool:
return True
def forward_cuda(
self,
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
x: torch.Tensor | None = None,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 1e-6,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
return torch.ops.vllm.mhc_pre_delayed_tilelang(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix,
x,
norm_weight,
norm_eps,
)
def forward_hip(
self,
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
x: torch.Tensor | None = None,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 1e-6,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
# The AITER delayed path drives mhc_pre_gemm_sqrsum against `residual`
# and folds no RMSNorm, so it cannot serve the model-entry broadcast
# (which projects a narrower `x`) or a fused norm. Both are handled by
# TileLang, or by the reference when TileLang is unavailable.
if (
x is None
and norm_weight is None
and _aiter_mhc_supported(residual, None, supports_norm=False)
):
return torch.ops.vllm.mhc_pre_delayed_aiter(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix,
)
if HAS_TILELANG_MHC:
return torch.ops.vllm.mhc_pre_delayed_tilelang(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix,
x,
norm_weight,
norm_eps,
)
return self.forward_native(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix,
x,
norm_weight,
norm_eps,
)
def forward_native(
self,
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
x: torch.Tensor | None = None,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 1e-6,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
post_mix, comb_mix, layer_input, next_pre_mix = (
mhc_kernels.mhc_pre_delayed_torch(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix=pre_mix,
x=x,
)
)
return (
post_mix,
comb_mix,
_apply_mhc_norm(layer_input, norm_weight, norm_eps),
next_pre_mix,
)
# --8<-- [start:mhc_post]
@CustomOp.register("mhc_post")
class MHCPostOp(CustomOp):
+16 -19
View File
@@ -17,15 +17,12 @@ from vllm.distributed import (
)
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.kernels.mhc.torch import (
mhc_post_torch,
mhc_pre_delayed_torch,
)
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mhc import MHCPostOp, MHCPreDelayedOp
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
@@ -248,11 +245,13 @@ class DeepseekV4DecoderLayer(nn.Module):
),
requires_grad=False,
)
self.mhc_pre_delayed = MHCPreDelayedOp()
self.mhc_post = MHCPostOp()
@staticmethod
def _hc_collapse(x: torch.Tensor, pre_mix: torch.Tensor) -> torch.Tensor:
"""Reference hyper-connection collapse used by DSpark on ROCm."""
return (pre_mix.unsqueeze(-1) * x.float()).sum(dim=1).to(x.dtype)
"""Hyper-connection collapse used by DSpark on ROCm."""
return torch.ops.vllm.hc_collapse_triton(x, pre_mix)
def forward(
self,
@@ -275,7 +274,7 @@ class DeepseekV4DecoderLayer(nn.Module):
# copies and the identity pre-mix selects copy 0.
assert self.hc_attn_fn_broadcast is not None
residual = x.unsqueeze(1).expand(-1, self.hc_mult, -1).contiguous()
post_mix, res_mix, x, attn_pre = mhc_pre_delayed_torch(
post_mix, res_mix, x, attn_pre = self.mhc_pre_delayed(
residual,
self.hc_attn_fn_broadcast,
self.hc_attn_scale,
@@ -289,7 +288,7 @@ class DeepseekV4DecoderLayer(nn.Module):
)
else:
residual = x
post_mix, res_mix, x, attn_pre = mhc_pre_delayed_torch(
post_mix, res_mix, x, attn_pre = self.mhc_pre_delayed(
residual,
self.hc_attn_fn,
self.hc_attn_scale,
@@ -302,7 +301,7 @@ class DeepseekV4DecoderLayer(nn.Module):
pre_mix=pre_mix,
)
else:
residual = mhc_post_torch(x, residual, post_mix, res_mix)
residual = self.mhc_post(x, residual, post_mix, res_mix)
if self.engram is not None and engram_hashes is not None:
# Engram injection happens between the previous sublayer's
# post and this block's pre, on the full hc stream, so the
@@ -312,7 +311,7 @@ class DeepseekV4DecoderLayer(nn.Module):
engram_hashes[:, self.engram.layer_hash_index],
engram_mask,
)
post_mix, res_mix, x, attn_pre = mhc_pre_delayed_torch(
post_mix, res_mix, x, attn_pre = self.mhc_pre_delayed(
residual,
self.hc_attn_fn,
self.hc_attn_scale,
@@ -333,8 +332,8 @@ class DeepseekV4DecoderLayer(nn.Module):
if self.use_sequence_parallel:
x = sp_reduce_scatter(x)
residual = mhc_post_torch(x, residual, post_mix, res_mix)
post_mix, res_mix, x, ffn_pre = mhc_pre_delayed_torch(
residual = self.mhc_post(x, residual, post_mix, res_mix)
post_mix, res_mix, x, ffn_pre = self.mhc_pre_delayed(
residual,
self.hc_ffn_fn,
self.hc_ffn_scale,
@@ -453,6 +452,8 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
else:
self.norm = PPMissingLayer()
self.mhc_post = MHCPostOp()
spec_config = vllm_config.speculative_config
needs_mtp_hidden_states = spec_config is not None and (
spec_config.use_eagle() or spec_config.uses_draft_model()
@@ -603,7 +604,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
)
if idx + 1 in self.aux_hidden_state_layers:
# Reconstruct the aux hidden state for draft models
aux_recon = mhc_post_torch(hidden_states, residual, post_mix, res_mix)
aux_recon = self.mhc_post(hidden_states, residual, post_mix, res_mix)
aux_hidden_state = aux_recon.mean(dim=1)
if self.use_sequence_parallel:
aux_hidden_state = sp_all_gather(aux_hidden_state)[:full_num_tokens]
@@ -614,7 +615,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
if self.end_layer in self.aux_hidden_state_layers:
hidden_states = final_aux_recon
else:
hidden_states = mhc_post_torch(
hidden_states = self.mhc_post(
hidden_states, residual, post_mix, res_mix
)
@@ -635,11 +636,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
# mixes — the mix the reference applies via
# ``last_layer.hc_pre(h, pre_mix)`` (v4.1 has no learned hc_head).
assert pre_mix is not None
hidden_states = (
(pre_mix.unsqueeze(-1) * hidden_states.float())
.sum(dim=1)
.to(hidden_states.dtype)
)
hidden_states = torch.ops.vllm.hc_collapse_triton(hidden_states, pre_mix)
hidden_states = self.norm(hidden_states)
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states