[ROCm][DSV4.1][Perf] Use AITER mHC for the delayed pre block (#56503)
Signed-off-by: Fangzhou Ai <fangzhou@semianalysis.ai> Co-authored-by: Fangzhou Ai <fangzhou@semianalysis.ai>
This commit is contained in:
co-authored by
Fangzhou Ai
parent
9d88ceb026
commit
46d2b23ac5
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.mhc import (
|
||||
HAS_AITER_MHC_PRE_NORM,
|
||||
HAS_TILELANG_MHC,
|
||||
MHCFusedPostPreOp,
|
||||
MHCPreDelayedOp,
|
||||
MHCPreOp,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.model import (
|
||||
@@ -697,6 +698,100 @@ def test_mhc_fused_rocm_fallback_applies_norm(monkeypatch):
|
||||
torch.testing.assert_close(out[3], expected_layer_input)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_rocm() and HAS_AITER_MHC),
|
||||
reason="AITER mHC required",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 2, 7, 128, 1024])
|
||||
@pytest.mark.parametrize("carried", [False, True])
|
||||
def test_mhc_pre_delayed_rocm_aiter(num_tokens, carried):
|
||||
"""AITER must reproduce the delayed reference on both seam variants.
|
||||
|
||||
``num_tokens`` spans the split-k choices AITER makes for the projection,
|
||||
since the pre-mix is recovered from that unreduced output.
|
||||
"""
|
||||
set_random_seed(0)
|
||||
hc_mult, hidden_size = 4, 5120
|
||||
residual, fn, hc_scale, hc_base, _ = _rocm_mhc_inputs(
|
||||
num_tokens=num_tokens, hidden_size=hidden_size, hc_mult=hc_mult
|
||||
)
|
||||
pre_mix = (
|
||||
torch.rand(num_tokens, hc_mult, dtype=torch.float32, device=DEVICE) + 0.5
|
||||
if carried
|
||||
else None
|
||||
)
|
||||
rms_eps = hc_pre_eps = hc_sinkhorn_eps = 1e-6
|
||||
args = (
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
1.0,
|
||||
20,
|
||||
)
|
||||
|
||||
expected = mhc_pre_delayed_torch(*args, pre_mix=pre_mix)
|
||||
actual = object.__new__(MHCPreDelayedOp).forward_hip(*args, pre_mix=pre_mix)
|
||||
|
||||
for i in (0, 1, 3):
|
||||
torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3)
|
||||
# The collapse is the same FP32 multiply-and-sum in both paths.
|
||||
torch.testing.assert_close(actual[2], expected[2], atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_rocm() and HAS_AITER_MHC),
|
||||
reason="AITER mHC required",
|
||||
)
|
||||
def test_mhc_pre_delayed_rocm_aiter_declines_unsupported(monkeypatch):
|
||||
"""The broadcast seam and a fused norm must not take the AITER path.
|
||||
|
||||
Neither is expressible with AITER's pre kernels: the broadcast projects a
|
||||
narrower ``x``, and ``mhc_pre_gemm_sqrsum`` folds no RMSNorm. What is
|
||||
asserted here is the routing decision, which is what this gate owns; the
|
||||
numerics of whichever fallback it lands on are covered by
|
||||
``test_deepseek_v41_mhc_pre_delayed``.
|
||||
"""
|
||||
set_random_seed(0)
|
||||
hc_mult, hidden_size = 4, 5120
|
||||
residual, fn, hc_scale, hc_base, norm_weight = _rocm_mhc_inputs(
|
||||
num_tokens=4, hidden_size=hidden_size, hc_mult=hc_mult
|
||||
)
|
||||
args = (residual, fn, hc_scale, hc_base, 1e-6, 1e-6, 1e-6, 1.0, 20)
|
||||
op = object.__new__(MHCPreDelayedOp)
|
||||
|
||||
aiter_op = torch.ops.vllm.mhc_pre_delayed_aiter
|
||||
took_aiter = False
|
||||
|
||||
def spy(*spy_args, **spy_kwargs):
|
||||
nonlocal took_aiter
|
||||
took_aiter = True
|
||||
return aiter_op(*spy_args, **spy_kwargs)
|
||||
|
||||
monkeypatch.setattr(torch.ops.vllm, "mhc_pre_delayed_aiter", spy)
|
||||
|
||||
x = residual[:, 0].contiguous()
|
||||
broadcast_fn = fn.view(-1, hc_mult, hidden_size).sum(1)
|
||||
broadcast_residual = x.unsqueeze(1).expand(-1, hc_mult, -1).contiguous()
|
||||
expected = mhc_pre_delayed_torch(broadcast_residual, broadcast_fn, *args[2:], x=x)
|
||||
actual = op.forward_hip(broadcast_residual, broadcast_fn, *args[2:], x=x)
|
||||
assert not took_aiter, "the broadcast seam must not reach AITER"
|
||||
for i in range(4):
|
||||
torch.testing.assert_close(actual[i], expected[i], atol=1e-4, rtol=1e-3)
|
||||
|
||||
op.forward_hip(*args, norm_weight=norm_weight, norm_eps=1e-6)
|
||||
assert not took_aiter, "a fused norm must not reach AITER"
|
||||
|
||||
# Positive control: the same inputs without a norm do take the AITER path,
|
||||
# so the two declines above are the gate discriminating rather than the
|
||||
# path being unavailable in this environment.
|
||||
op.forward_hip(*args)
|
||||
assert took_aiter
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_rocm() and HAS_AITER_MHC and HAS_AITER_MHC_PRE_NORM),
|
||||
reason="AITER mHC with fused RMSNorm required",
|
||||
|
||||
@@ -3599,6 +3599,137 @@ class rocm_aiter_ops:
|
||||
layer_input.view(*outer_shape, hidden_size),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def mhc_pre_delayed(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""mHC pre using the pre-mix carried from the previous sublayer.
|
||||
|
||||
Same gates as :meth:`mhc_pre`, but the stream collapse applies the
|
||||
caller's ``pre_mix`` instead of the one computed here, and the one
|
||||
computed here is returned for the next sublayer seam. AITER has no
|
||||
single kernel for that shape, so drive its two stages directly:
|
||||
``mhc_pre_big_fuse`` still produces the post and comb gates (including
|
||||
every sinkhorn iteration), while the pre gate is recovered from the
|
||||
same split-k GEMM output and the collapse is done by the Triton
|
||||
kernel. ``mhc_pre_big_fuse`` also writes a collapse we do not use;
|
||||
that redundant store is the price of not having a native delayed
|
||||
kernel, and is small next to the ~140 launches it replaces.
|
||||
|
||||
Returns:
|
||||
post_mix: shape (..., hc_mult, 1), dtype torch.float32
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
next_pre_mix: shape (..., hc_mult), dtype torch.float32
|
||||
"""
|
||||
from aiter.ops.mhc import (
|
||||
get_mhc_pre_splitk,
|
||||
mhc_pre_big_fuse,
|
||||
mhc_pre_gemm_sqrsum,
|
||||
)
|
||||
|
||||
assert residual.dtype == torch.bfloat16
|
||||
assert fn.dtype == torch.float32
|
||||
assert hc_scale.dtype == torch.float32
|
||||
assert hc_base.dtype == torch.float32
|
||||
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
hc_mult3 = hc_mult * 2 + hc_mult * hc_mult
|
||||
hc_hidden_size = hc_mult * hidden_size
|
||||
|
||||
assert fn.shape == (hc_mult3, hc_hidden_size)
|
||||
assert hc_scale.shape == (3,)
|
||||
assert hc_base.shape == (hc_mult3,)
|
||||
|
||||
outer_shape = residual.shape[:-2]
|
||||
residual_flat = residual.view(-1, hc_mult, hidden_size)
|
||||
num_tokens = residual_flat.shape[0]
|
||||
device = residual_flat.device
|
||||
|
||||
if num_tokens == 0:
|
||||
return (
|
||||
torch.empty(0, hc_mult, 1, dtype=torch.float32, device=device),
|
||||
torch.empty(0, hc_mult, hc_mult, dtype=torch.float32, device=device),
|
||||
torch.empty(0, hidden_size, dtype=torch.bfloat16, device=device),
|
||||
torch.empty(0, hc_mult, dtype=torch.float32, device=device),
|
||||
)
|
||||
|
||||
pre_mix_flat = None if pre_mix is None else pre_mix.view(-1, hc_mult)
|
||||
|
||||
# AITER's Python wrappers allocate without explicit device arguments.
|
||||
with torch.device(device):
|
||||
splitk, tile_k = get_mhc_pre_splitk(num_tokens, hc_hidden_size)
|
||||
# AITER pads the GEMM output to a multiple of 32 columns.
|
||||
gemm_pad = torch.empty(
|
||||
splitk,
|
||||
num_tokens,
|
||||
(hc_mult3 + 31) // 32 * 32,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
gemm_out = gemm_pad[:, :, :hc_mult3]
|
||||
sqrsum = torch.empty(splitk, num_tokens, dtype=torch.float32, device=device)
|
||||
mhc_pre_gemm_sqrsum(gemm_out, sqrsum, residual_flat, fn, tile_k, 0)
|
||||
|
||||
post_mix = torch.empty(
|
||||
num_tokens, hc_mult, 1, dtype=torch.float32, device=device
|
||||
)
|
||||
comb_mix = torch.empty(
|
||||
num_tokens, hc_mult, hc_mult, dtype=torch.float32, device=device
|
||||
)
|
||||
unused_collapse = torch.empty(
|
||||
num_tokens, hidden_size, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
mhc_pre_big_fuse(
|
||||
post_mix,
|
||||
comb_mix,
|
||||
unused_collapse,
|
||||
gemm_out,
|
||||
sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
residual_flat,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
|
||||
next_pre_mix = torch.ops.vllm.mhc_pre_mix_triton(
|
||||
gemm_out,
|
||||
sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
hc_mult,
|
||||
hc_hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
)
|
||||
|
||||
if pre_mix_flat is None:
|
||||
# Model entry selects residual stream zero.
|
||||
layer_input = residual_flat[:, 0]
|
||||
else:
|
||||
layer_input = torch.ops.vllm.hc_collapse_triton(residual_flat, pre_mix_flat)
|
||||
|
||||
return (
|
||||
post_mix.view(*outer_shape, hc_mult, 1),
|
||||
comb_mix.view(*outer_shape, hc_mult, hc_mult),
|
||||
layer_input.view(*outer_shape, hidden_size),
|
||||
next_pre_mix.view(*outer_shape, hc_mult),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def hc_head(
|
||||
hs_flat: torch.Tensor,
|
||||
|
||||
@@ -12,6 +12,7 @@ __all__ = [
|
||||
"mhc_fused_post_pre_cuda",
|
||||
"hc_head_fused_kernel_cuda",
|
||||
"mhc_pre_aiter",
|
||||
"mhc_pre_delayed_aiter",
|
||||
"mhc_post_aiter",
|
||||
"mhc_fused_post_pre_aiter",
|
||||
"hc_head_fused_aiter",
|
||||
@@ -26,6 +27,7 @@ __all__ = [
|
||||
"mhc_post_torch",
|
||||
"mhc_fused_post_pre_torch",
|
||||
"hc_head_fused_torch",
|
||||
"mhc_pre_mix_triton",
|
||||
"mhc_pre_triton",
|
||||
"mhc_post_triton",
|
||||
"mhc_fused_post_pre_triton",
|
||||
|
||||
@@ -104,6 +104,90 @@ def _mhc_pre_aiter_fake(
|
||||
return post_mix, comb_mix, layer_input
|
||||
|
||||
|
||||
def mhc_pre_delayed_aiter(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""mHC pre with the pre-mix carried in from the previous sublayer.
|
||||
|
||||
Matches ``mhc_pre_delayed_torch``: the stream collapse uses *pre_mix*
|
||||
rather than the gate computed here, and that gate is returned as the
|
||||
pre-mix for the next sublayer seam.
|
||||
|
||||
Args:
|
||||
residual: shape (..., hc_mult, hidden_size), dtype torch.bfloat16
|
||||
fn: shape (hc_mult3, hc_mult * hidden_size), dtype torch.float32
|
||||
hc_scale: shape (3,), dtype torch.float32
|
||||
hc_base: shape (hc_mult3,), dtype torch.float32
|
||||
rms_eps: RMS normalization epsilon
|
||||
hc_pre_eps: pre-mix epsilon
|
||||
hc_sinkhorn_eps: sinkhorn epsilon
|
||||
hc_post_mult_value: post-mix multiplier value
|
||||
sinkhorn_repeat: number of sinkhorn iterations
|
||||
pre_mix: shape (..., hc_mult) from the previous sublayer, or None at
|
||||
model entry to select residual stream zero.
|
||||
|
||||
Returns:
|
||||
post_mix: shape (..., hc_mult, 1), dtype torch.float32
|
||||
comb_mix: shape (..., hc_mult, hc_mult), dtype torch.float32
|
||||
layer_input: shape (..., hidden_size), dtype torch.bfloat16
|
||||
next_pre_mix: shape (..., hc_mult), dtype torch.float32
|
||||
"""
|
||||
hidden_size = residual.shape[-1]
|
||||
assert hidden_size % 256 == 0
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.mhc_pre_delayed(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix,
|
||||
)
|
||||
|
||||
|
||||
def _mhc_pre_delayed_aiter_fake(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
hc_mult = residual.shape[-2]
|
||||
hidden_size = residual.shape[-1]
|
||||
outer_shape = residual.shape[:-2]
|
||||
return (
|
||||
torch.empty(
|
||||
*outer_shape, hc_mult, 1, dtype=torch.float32, device=residual.device
|
||||
),
|
||||
torch.empty(
|
||||
*outer_shape, hc_mult, hc_mult, dtype=torch.float32, device=residual.device
|
||||
),
|
||||
torch.empty(
|
||||
*outer_shape, hidden_size, dtype=torch.bfloat16, device=residual.device
|
||||
),
|
||||
torch.empty(*outer_shape, hc_mult, dtype=torch.float32, device=residual.device),
|
||||
)
|
||||
|
||||
|
||||
def mhc_post_aiter(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
@@ -226,6 +310,12 @@ direct_register_custom_op(
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_pre_aiter_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_pre_delayed_aiter",
|
||||
op_func=mhc_pre_delayed_aiter,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_pre_delayed_aiter_fake,
|
||||
)
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_post_aiter",
|
||||
op_func=mhc_post_aiter,
|
||||
|
||||
@@ -141,6 +141,129 @@ def _hc_collapse_triton_fake(x: Tensor, pre_mix: Tensor) -> Tensor:
|
||||
return torch.empty(x.shape[0], x.shape[2], dtype=x.dtype, device=x.device)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _mhc_pre_mix_kernel(
|
||||
gemm_ptr,
|
||||
sqrsum_ptr,
|
||||
hc_scale_ptr,
|
||||
hc_base_ptr,
|
||||
out_ptr,
|
||||
splitk,
|
||||
hc_mult: tl.constexpr,
|
||||
gemm_stride_k,
|
||||
gemm_stride_t,
|
||||
gemm_stride_j,
|
||||
sqrsum_stride_k,
|
||||
sqrsum_stride_t,
|
||||
out_stride_t,
|
||||
out_stride_j,
|
||||
inv_hc_hidden,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
SPLITK_BLOCK: tl.constexpr,
|
||||
HC_BLOCK: tl.constexpr,
|
||||
):
|
||||
"""Recover the pre-mix gate from a split-k mHC pre GEMM output."""
|
||||
token_idx = tl.program_id(0).to(tl.int64)
|
||||
ks = tl.arange(0, SPLITK_BLOCK)
|
||||
js = tl.arange(0, HC_BLOCK)
|
||||
kmask = ks < splitk
|
||||
jmask = js < hc_mult
|
||||
|
||||
# Only the first hc_mult GEMM columns feed the pre gate.
|
||||
gemm = tl.load(
|
||||
gemm_ptr
|
||||
+ ks[:, None] * gemm_stride_k
|
||||
+ token_idx * gemm_stride_t
|
||||
+ js[None, :] * gemm_stride_j,
|
||||
mask=kmask[:, None] & jmask[None, :],
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
mixes = tl.sum(gemm, 0)
|
||||
|
||||
sqrsum = tl.load(
|
||||
sqrsum_ptr + ks * sqrsum_stride_k + token_idx * sqrsum_stride_t,
|
||||
mask=kmask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
rstd = tl.rsqrt(tl.sum(sqrsum, 0) * inv_hc_hidden + rms_eps)
|
||||
|
||||
scale = tl.load(hc_scale_ptr).to(tl.float32)
|
||||
base = tl.load(hc_base_ptr + js, mask=jmask, other=0.0).to(tl.float32)
|
||||
|
||||
pre = tl.sigmoid(mixes * rstd * scale + base) + hc_pre_eps
|
||||
tl.store(out_ptr + token_idx * out_stride_t + js * out_stride_j, pre, mask=jmask)
|
||||
|
||||
|
||||
def mhc_pre_mix_triton(
|
||||
gemm_out: Tensor,
|
||||
sqrsum: Tensor,
|
||||
hc_scale: Tensor,
|
||||
hc_base: Tensor,
|
||||
hc_mult: int,
|
||||
hc_hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
) -> Tensor:
|
||||
"""Pre-mix gate for the delayed mHC pre, from AITER's split-k GEMM output.
|
||||
|
||||
AITER's ``mhc_pre_big_fuse`` consumes the unreduced ``[splitk, tokens,
|
||||
hc_mult3]`` GEMM output and the matching row square-sums, but only returns
|
||||
the post and comb gates. The delayed formulation also needs the pre gate,
|
||||
to carry into the next sublayer seam. It is the same slice of the same
|
||||
numbers, so recover it here rather than repeating the projection.
|
||||
"""
|
||||
assert gemm_out.ndim == 3 and gemm_out.dtype == torch.float32
|
||||
assert sqrsum.ndim == 2 and sqrsum.dtype == torch.float32
|
||||
splitk, num_tokens = gemm_out.shape[0], gemm_out.shape[1]
|
||||
assert sqrsum.shape == (splitk, num_tokens)
|
||||
|
||||
out = torch.empty(num_tokens, hc_mult, dtype=torch.float32, device=gemm_out.device)
|
||||
if num_tokens == 0:
|
||||
return out
|
||||
|
||||
_mhc_pre_mix_kernel[(num_tokens,)](
|
||||
gemm_out,
|
||||
sqrsum,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
splitk,
|
||||
hc_mult,
|
||||
gemm_out.stride(0),
|
||||
gemm_out.stride(1),
|
||||
gemm_out.stride(2),
|
||||
sqrsum.stride(0),
|
||||
sqrsum.stride(1),
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
1.0 / hc_hidden_size,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
SPLITK_BLOCK=triton.next_power_of_2(splitk),
|
||||
HC_BLOCK=triton.next_power_of_2(hc_mult),
|
||||
num_warps=1,
|
||||
# Match the separate FP32 multiply and add in the Torch reference.
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _mhc_pre_mix_triton_fake(
|
||||
gemm_out: Tensor,
|
||||
sqrsum: Tensor,
|
||||
hc_scale: Tensor,
|
||||
hc_base: Tensor,
|
||||
hc_mult: int,
|
||||
hc_hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
) -> Tensor:
|
||||
return torch.empty(
|
||||
gemm_out.shape[1], hc_mult, dtype=torch.float32, device=gemm_out.device
|
||||
)
|
||||
|
||||
|
||||
def hc_head_reduce_triton_kernel(
|
||||
x: torch.Tensor,
|
||||
hc_fn: torch.Tensor,
|
||||
@@ -216,3 +339,11 @@ direct_register_custom_op(
|
||||
mutates_args=[],
|
||||
fake_impl=_hc_collapse_triton_fake,
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="mhc_pre_mix_triton",
|
||||
op_func=mhc_pre_mix_triton,
|
||||
mutates_args=[],
|
||||
fake_impl=_mhc_pre_mix_triton_fake,
|
||||
)
|
||||
|
||||
@@ -292,6 +292,161 @@ class MHCPreOp(CustomOp):
|
||||
)
|
||||
|
||||
|
||||
# --8<-- [start:mhc_pre_delayed]
|
||||
@CustomOp.register("mhc_pre_delayed")
|
||||
class MHCPreDelayedOp(CustomOp):
|
||||
"""MHC pre block using the pre-mix carried from the previous sublayer.
|
||||
|
||||
Same gates as :class:`MHCPreOp`, but the stream collapse applies the
|
||||
caller's ``pre_mix`` and this sublayer's pre-mix is returned for the next
|
||||
sublayer seam. Returns post_mix, comb_mix, layer_input, next_pre_mix.
|
||||
"""
|
||||
|
||||
# --8<-- [end:mhc_pre_delayed]
|
||||
@classmethod
|
||||
def enabled(cls) -> bool:
|
||||
return True
|
||||
|
||||
def forward_cuda(
|
||||
self,
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
x: torch.Tensor | None = None,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 1e-6,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return torch.ops.vllm.mhc_pre_delayed_tilelang(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix,
|
||||
x,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
|
||||
def forward_hip(
|
||||
self,
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
x: torch.Tensor | None = None,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 1e-6,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# The AITER delayed path drives mhc_pre_gemm_sqrsum against `residual`
|
||||
# and folds no RMSNorm, so it cannot serve the model-entry broadcast
|
||||
# (which projects a narrower `x`) or a fused norm. Both are handled by
|
||||
# TileLang, or by the reference when TileLang is unavailable.
|
||||
if (
|
||||
x is None
|
||||
and norm_weight is None
|
||||
and _aiter_mhc_supported(residual, None, supports_norm=False)
|
||||
):
|
||||
return torch.ops.vllm.mhc_pre_delayed_aiter(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix,
|
||||
)
|
||||
if HAS_TILELANG_MHC:
|
||||
return torch.ops.vllm.mhc_pre_delayed_tilelang(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix,
|
||||
x,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
return self.forward_native(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix,
|
||||
x,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
x: torch.Tensor | None = None,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 1e-6,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
post_mix, comb_mix, layer_input, next_pre_mix = (
|
||||
mhc_kernels.mhc_pre_delayed_torch(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix=pre_mix,
|
||||
x=x,
|
||||
)
|
||||
)
|
||||
return (
|
||||
post_mix,
|
||||
comb_mix,
|
||||
_apply_mhc_norm(layer_input, norm_weight, norm_eps),
|
||||
next_pre_mix,
|
||||
)
|
||||
|
||||
|
||||
# --8<-- [start:mhc_post]
|
||||
@CustomOp.register("mhc_post")
|
||||
class MHCPostOp(CustomOp):
|
||||
|
||||
@@ -17,15 +17,12 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.torch import (
|
||||
mhc_post_torch,
|
||||
mhc_pre_delayed_torch,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mhc import MHCPostOp, MHCPreDelayedOp
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
@@ -248,11 +245,13 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.mhc_pre_delayed = MHCPreDelayedOp()
|
||||
self.mhc_post = MHCPostOp()
|
||||
|
||||
@staticmethod
|
||||
def _hc_collapse(x: torch.Tensor, pre_mix: torch.Tensor) -> torch.Tensor:
|
||||
"""Reference hyper-connection collapse used by DSpark on ROCm."""
|
||||
return (pre_mix.unsqueeze(-1) * x.float()).sum(dim=1).to(x.dtype)
|
||||
"""Hyper-connection collapse used by DSpark on ROCm."""
|
||||
return torch.ops.vllm.hc_collapse_triton(x, pre_mix)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -275,7 +274,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
# copies and the identity pre-mix selects copy 0.
|
||||
assert self.hc_attn_fn_broadcast is not None
|
||||
residual = x.unsqueeze(1).expand(-1, self.hc_mult, -1).contiguous()
|
||||
post_mix, res_mix, x, attn_pre = mhc_pre_delayed_torch(
|
||||
post_mix, res_mix, x, attn_pre = self.mhc_pre_delayed(
|
||||
residual,
|
||||
self.hc_attn_fn_broadcast,
|
||||
self.hc_attn_scale,
|
||||
@@ -289,7 +288,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
)
|
||||
else:
|
||||
residual = x
|
||||
post_mix, res_mix, x, attn_pre = mhc_pre_delayed_torch(
|
||||
post_mix, res_mix, x, attn_pre = self.mhc_pre_delayed(
|
||||
residual,
|
||||
self.hc_attn_fn,
|
||||
self.hc_attn_scale,
|
||||
@@ -302,7 +301,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
pre_mix=pre_mix,
|
||||
)
|
||||
else:
|
||||
residual = mhc_post_torch(x, residual, post_mix, res_mix)
|
||||
residual = self.mhc_post(x, residual, post_mix, res_mix)
|
||||
if self.engram is not None and engram_hashes is not None:
|
||||
# Engram injection happens between the previous sublayer's
|
||||
# post and this block's pre, on the full hc stream, so the
|
||||
@@ -312,7 +311,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
engram_hashes[:, self.engram.layer_hash_index],
|
||||
engram_mask,
|
||||
)
|
||||
post_mix, res_mix, x, attn_pre = mhc_pre_delayed_torch(
|
||||
post_mix, res_mix, x, attn_pre = self.mhc_pre_delayed(
|
||||
residual,
|
||||
self.hc_attn_fn,
|
||||
self.hc_attn_scale,
|
||||
@@ -333,8 +332,8 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
if self.use_sequence_parallel:
|
||||
x = sp_reduce_scatter(x)
|
||||
|
||||
residual = mhc_post_torch(x, residual, post_mix, res_mix)
|
||||
post_mix, res_mix, x, ffn_pre = mhc_pre_delayed_torch(
|
||||
residual = self.mhc_post(x, residual, post_mix, res_mix)
|
||||
post_mix, res_mix, x, ffn_pre = self.mhc_pre_delayed(
|
||||
residual,
|
||||
self.hc_ffn_fn,
|
||||
self.hc_ffn_scale,
|
||||
@@ -453,6 +452,8 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
else:
|
||||
self.norm = PPMissingLayer()
|
||||
|
||||
self.mhc_post = MHCPostOp()
|
||||
|
||||
spec_config = vllm_config.speculative_config
|
||||
needs_mtp_hidden_states = spec_config is not None and (
|
||||
spec_config.use_eagle() or spec_config.uses_draft_model()
|
||||
@@ -603,7 +604,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
)
|
||||
if idx + 1 in self.aux_hidden_state_layers:
|
||||
# Reconstruct the aux hidden state for draft models
|
||||
aux_recon = mhc_post_torch(hidden_states, residual, post_mix, res_mix)
|
||||
aux_recon = self.mhc_post(hidden_states, residual, post_mix, res_mix)
|
||||
aux_hidden_state = aux_recon.mean(dim=1)
|
||||
if self.use_sequence_parallel:
|
||||
aux_hidden_state = sp_all_gather(aux_hidden_state)[:full_num_tokens]
|
||||
@@ -614,7 +615,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
if self.end_layer in self.aux_hidden_state_layers:
|
||||
hidden_states = final_aux_recon
|
||||
else:
|
||||
hidden_states = mhc_post_torch(
|
||||
hidden_states = self.mhc_post(
|
||||
hidden_states, residual, post_mix, res_mix
|
||||
)
|
||||
|
||||
@@ -635,11 +636,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
# mixes — the mix the reference applies via
|
||||
# ``last_layer.hc_pre(h, pre_mix)`` (v4.1 has no learned hc_head).
|
||||
assert pre_mix is not None
|
||||
hidden_states = (
|
||||
(pre_mix.unsqueeze(-1) * hidden_states.float())
|
||||
.sum(dim=1)
|
||||
.to(hidden_states.dtype)
|
||||
)
|
||||
hidden_states = torch.ops.vllm.hc_collapse_triton(hidden_states, pre_mix)
|
||||
hidden_states = self.norm(hidden_states)
|
||||
if len(aux_hidden_states) > 0:
|
||||
return hidden_states, aux_hidden_states
|
||||
|
||||
Reference in New Issue
Block a user