[New Model][ROCm] Add AMD support for DeepSeek V4 (#40871)
Signed-off-by: ganyi <ygan@amd.com> Signed-off-by: whx-sjtu <xiaowang990929@gmail.com> Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Signed-off-by: tjtanaavllm <tunjian.tan@amd.com> Co-authored-by: ganyi <ygan@amd.com> Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: tjtanaavllm <tunjian.tan@amd.com>
This commit is contained in:
co-authored by
ganyi
tjtanaa
tjtanaavllm
parent
2228fe6868
commit
628c436301
@@ -312,6 +312,21 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
As: torch.Tensor,
|
||||
Bs: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
if As.dtype != Bs.dtype:
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
_upcast_e8m0_to_fp32,
|
||||
)
|
||||
|
||||
if As.dtype == torch.float8_e8m0fnu:
|
||||
As = _upcast_e8m0_to_fp32(As).contiguous()
|
||||
else:
|
||||
As = As.to(torch.float32)
|
||||
|
||||
if Bs.dtype == torch.float8_e8m0fnu:
|
||||
Bs = _upcast_e8m0_to_fp32(Bs).contiguous()
|
||||
else:
|
||||
Bs = Bs.to(torch.float32)
|
||||
|
||||
out_dtype = self.config.out_dtype
|
||||
if self.use_triton:
|
||||
gemm_a8w8_blockscale_op = rocm_aiter_ops.triton_gemm_a8w8_blockscale
|
||||
|
||||
@@ -169,7 +169,9 @@ class SiluAndMulWithClamp(CustomOp):
|
||||
def __init__(self, swiglu_limit: float, *, compile_native: bool = True):
|
||||
super().__init__(compile_native=compile_native)
|
||||
self.swiglu_limit = float(swiglu_limit)
|
||||
if current_platform.is_cuda_alike() or current_platform.is_xpu():
|
||||
if current_platform.is_rocm():
|
||||
self._forward_method = self.forward_native
|
||||
elif current_platform.is_cuda_alike() or current_platform.is_xpu():
|
||||
self.op = torch.ops._C.silu_and_mul_with_clamp
|
||||
elif current_platform.is_cpu():
|
||||
self._forward_method = self.forward_native
|
||||
|
||||
@@ -300,6 +300,7 @@ class DeepseekCompressor(nn.Module):
|
||||
state_cache = self.state_cache.kv_cache
|
||||
# kv_state stored in first half, score_state stored in second half
|
||||
state_width = state_cache.shape[-1] // 2
|
||||
pdl_kwargs = {} if current_platform.is_rocm() else {"launch_pdl": False}
|
||||
|
||||
# Store the KV and score (with fused APE addition) in the state.
|
||||
# NOTE: PDL is disabled — both this kernel and _fused_kernel below
|
||||
@@ -324,7 +325,7 @@ class DeepseekCompressor(nn.Module):
|
||||
TRITON_BLOCK_SIZE=triton.next_power_of_2(kv.shape[-1]),
|
||||
STATE_WIDTH=state_width,
|
||||
COMPRESS_RATIO=self.compress_ratio,
|
||||
launch_pdl=False,
|
||||
**pdl_kwargs,
|
||||
)
|
||||
|
||||
# Fused: compress → RMSNorm → RoPE → FP8 quant → KV cache write.
|
||||
@@ -373,7 +374,7 @@ class DeepseekCompressor(nn.Module):
|
||||
SCALE_DIM=self._scale_dim,
|
||||
KV_BLOCK_STRIDE=kv_cache.stride(0),
|
||||
num_warps=self._num_warps,
|
||||
launch_pdl=False,
|
||||
**pdl_kwargs,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -28,6 +28,11 @@ from vllm.v1.attention.ops.deepseek_v4_ops import (
|
||||
fused_inv_rope_fp8_quant,
|
||||
fused_q_kv_rmsnorm,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
rocm_forward_decode_fallback,
|
||||
rocm_inv_rope_einsum,
|
||||
rocm_sparse_attn_prefill,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import (
|
||||
@@ -53,6 +58,7 @@ from vllm.model_executor.layers.quantization.input_quant_fp8 import (
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.multi_stream_utils import (
|
||||
execute_in_parallel,
|
||||
maybe_execute_in_parallel,
|
||||
@@ -198,8 +204,6 @@ class DeepseekV4MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
# Pick fp8_einsum recipe based on GPU arch:
|
||||
# SM90: FP32 block scales stay [g, r/128, d/128] → sfb_gran_mn=128
|
||||
# SM100: INT32 packed scales become [g, r, ...] → sfb_gran_mn=1
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
cap = current_platform.get_device_capability()
|
||||
assert cap is not None, "DeepseekV4 attention requires a CUDA device"
|
||||
self._einsum_recipe = (1, 128, 128) if cap.major <= 9 else (1, 1, 128)
|
||||
@@ -222,6 +226,7 @@ class DeepseekV4MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
+ 1 # 1B pad
|
||||
)
|
||||
|
||||
# Will be None on ROCm for now.
|
||||
self.aux_stream_list = mla_modules.aux_stream_list
|
||||
# [0]: GEMM start / post-GEMM event0. [1..3]: GEMM done events;
|
||||
# [1] doubles as post-GEMM event1. Reuse is safe: GEMM fully joins
|
||||
@@ -303,6 +308,19 @@ class DeepseekV4MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
)
|
||||
o = o_padded[:, : self.n_local_heads, :]
|
||||
|
||||
# Keep ROCm on the BF16 reference wo_a path util kernel ready.
|
||||
if current_platform.is_rocm():
|
||||
z = rocm_inv_rope_einsum(
|
||||
self.rotary_emb,
|
||||
o,
|
||||
positions,
|
||||
self.rope_head_dim,
|
||||
self.n_local_groups,
|
||||
self.o_lora_rank,
|
||||
self.wo_a,
|
||||
)
|
||||
return self.wo_b(z.flatten(1))
|
||||
|
||||
# O projection: inverse RoPE + FP8 quant + einsum + wo_b
|
||||
o_fp8, o_scale = fused_inv_rope_fp8_quant(
|
||||
o,
|
||||
@@ -336,12 +354,15 @@ class DeepseekV4MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
return self.wo_b(z.flatten(1))
|
||||
|
||||
def attn_gemm_parallel_execute(self, hidden_states) -> tuple[Any, ...]:
|
||||
assert self.aux_stream_list is not None
|
||||
assert len(self.aux_stream_list) >= 3
|
||||
aux_streams = self.aux_stream_list
|
||||
if aux_streams is not None:
|
||||
assert len(aux_streams) >= 3
|
||||
aux_streams = aux_streams[:3]
|
||||
|
||||
# fused_wqa_wkv (heaviest) on default; the three lighter input GEMMs
|
||||
# on aux streams 0..2 when their owning module exists. ln_events[0]
|
||||
# is the fan-out start event; ln_events[1..3] are per-aux done events.
|
||||
# On ROCm, aux_streams is None and execute_in_parallel runs serially.
|
||||
aux_fns: list[Callable[[], Any] | None] = [None, None, None]
|
||||
|
||||
if self.compressor is not None:
|
||||
@@ -385,7 +406,7 @@ class DeepseekV4MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
aux_fns,
|
||||
self.ln_events[0],
|
||||
self.ln_events[1:4],
|
||||
self.aux_stream_list[:3],
|
||||
aux_streams,
|
||||
enable=hidden_states.shape[0]
|
||||
<= envs.VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD,
|
||||
)
|
||||
@@ -419,8 +440,9 @@ class DeepseekV4MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
# downstream reads q on default). Indexer/compressor go on aux for
|
||||
# overlap with default's GEMM + cache write.
|
||||
if self.indexer is not None:
|
||||
assert self.aux_stream_list is not None
|
||||
aux_stream = self.aux_stream_list[0]
|
||||
aux_stream = (
|
||||
self.aux_stream_list[0] if self.aux_stream_list is not None else None
|
||||
)
|
||||
indexer = self.indexer
|
||||
# Local ref so the closure keeps a non-None type for mypy.
|
||||
assert self.compressor is not None
|
||||
@@ -448,8 +470,9 @@ class DeepseekV4MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
)
|
||||
elif self.compressor is not None:
|
||||
# wq_b + kv_insert on default, compressor on aux.
|
||||
assert self.aux_stream_list is not None
|
||||
aux_stream = self.aux_stream_list[0]
|
||||
aux_stream = (
|
||||
self.aux_stream_list[0] if self.aux_stream_list is not None else None
|
||||
)
|
||||
compressor = self.compressor
|
||||
|
||||
def wq_b_kv_insert() -> torch.Tensor:
|
||||
@@ -668,7 +691,7 @@ class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase):
|
||||
vllm_config.scheduler_config.max_num_batched_tokens
|
||||
)
|
||||
self.max_model_len = vllm_config.model_config.max_model_len
|
||||
# DeepseekV4 only supports fp8 kv-cache format for now
|
||||
# DeepseekV4 only supports fp8 kv-cache format for now.
|
||||
kv_cache_dtype = cache_config.cache_dtype if cache_config is not None else "fp8"
|
||||
|
||||
assert kv_cache_dtype.startswith("fp8"), (
|
||||
@@ -816,6 +839,25 @@ class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase):
|
||||
swa_indices = swa_metadata.decode_swa_indices
|
||||
swa_lens = swa_metadata.decode_swa_lens
|
||||
|
||||
if current_platform.is_rocm():
|
||||
rocm_forward_decode_fallback(
|
||||
q=q,
|
||||
kv_cache=kv_cache,
|
||||
swa_k_cache=self.swa_cache_layer.kv_cache,
|
||||
swa_only=swa_only,
|
||||
topk_indices=topk_indices,
|
||||
topk_lens=topk_lens,
|
||||
swa_indices=swa_indices,
|
||||
swa_lens=swa_lens,
|
||||
attn_sink=self.attn_sink,
|
||||
scale=self.scale,
|
||||
head_dim=self.head_dim,
|
||||
nope_head_dim=self.nope_head_dim,
|
||||
rope_head_dim=self.rope_head_dim,
|
||||
output=output,
|
||||
)
|
||||
return
|
||||
|
||||
# We treat queries in the same seq as different queries
|
||||
# and later we only attend by generated indices.
|
||||
# q arrives pre-padded to self.padded_heads by the outer wrapper.
|
||||
@@ -980,15 +1022,27 @@ class DeepseekV4MLAAttention(nn.Module, AttentionLayerBase):
|
||||
N,
|
||||
)
|
||||
|
||||
output_chunk, _, _ = flash_mla_sparse_fwd(
|
||||
q=q[query_start:query_end],
|
||||
kv=kv.view(-1, 1, q.shape[-1]),
|
||||
indices=combined_indices.unsqueeze(1),
|
||||
sm_scale=self.scale,
|
||||
attn_sink=self.attn_sink,
|
||||
topk_length=combined_lens,
|
||||
out=output[query_start:query_end],
|
||||
)
|
||||
if current_platform.is_rocm():
|
||||
rocm_sparse_attn_prefill(
|
||||
q=q[query_start:query_end],
|
||||
kv=kv.view(-1, 1, q.shape[-1]),
|
||||
indices=combined_indices.unsqueeze(1),
|
||||
topk_length=combined_lens,
|
||||
scale=self.scale,
|
||||
head_dim=self.head_dim,
|
||||
attn_sink=self.attn_sink,
|
||||
output=output[query_start:query_end],
|
||||
)
|
||||
else:
|
||||
output_chunk, _, _ = flash_mla_sparse_fwd(
|
||||
q=q[query_start:query_end],
|
||||
kv=kv.view(-1, 1, q.shape[-1]),
|
||||
indices=combined_indices.unsqueeze(1),
|
||||
sm_scale=self.scale,
|
||||
attn_sink=self.attn_sink,
|
||||
topk_length=combined_lens,
|
||||
out=output[query_start:query_end],
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
|
||||
|
||||
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
FusedMoEQuantDesc,
|
||||
RoutingMethodType,
|
||||
mxfp4_mxfp8_moe_quant_config,
|
||||
mxfp4_w4a8_moe_quant_config,
|
||||
mxfp4_w4a16_moe_quant_config,
|
||||
@@ -64,6 +65,8 @@ class Mxfp4MoeBackend(Enum):
|
||||
MARLIN = "MARLIN"
|
||||
# ROCm AITER backends
|
||||
AITER_MXFP4_BF16 = "AITER_MXFP4_BF16" # W4A16: CK kernel
|
||||
# Keep the legacy name as an alias while the ROCm split backend rename settles.
|
||||
AITER = "AITER_MXFP4_BF16"
|
||||
AITER_MXFP4_FP8 = "AITER_MXFP4_FP8" # W4A8: triton kernel
|
||||
# Triton
|
||||
TRITON = "TRITON"
|
||||
@@ -253,6 +256,8 @@ def _get_priority_backends() -> list[Mxfp4MoeBackend]:
|
||||
TRTLLM MXFP8; SM90 falls through to Triton_unfused or Marlin (the
|
||||
backend-level ``is_supported_config`` check filters by device capability).
|
||||
"""
|
||||
if current_platform.is_rocm():
|
||||
return [Mxfp4MoeBackend.AITER_MXFP4_BF16]
|
||||
_AVAILABLE_BACKENDS = [
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
Mxfp4MoeBackend.DEEPGEMM_MXFP4,
|
||||
@@ -543,8 +548,22 @@ def select_deepseek_v4_mxfp4_moe_backend(
|
||||
activation_format,
|
||||
)
|
||||
|
||||
# DeepSeek-V4 on ROCm is more accurate with the unfused Triton MXFP4 path
|
||||
# than the default AITER path. Prefer Triton-unfused for this routing mode,
|
||||
# while keeping AITER as a fallback if Triton-unfused rejects the config.
|
||||
if (
|
||||
current_platform.is_rocm()
|
||||
and config.routing_method == RoutingMethodType.DeepseekV4
|
||||
):
|
||||
priority_backends = [
|
||||
Mxfp4MoeBackend.TRITON_UNFUSED,
|
||||
Mxfp4MoeBackend.AITER_MXFP4_BF16,
|
||||
]
|
||||
else:
|
||||
priority_backends = _get_priority_backends()
|
||||
|
||||
# Iterate priority backends: TRTLLM MXFP8, then Triton.
|
||||
for backend in _get_priority_backends():
|
||||
for backend in priority_backends:
|
||||
activation_key = _backend_activation_key(backend)
|
||||
for k_cls in backend_to_kernel_cls(backend):
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
@@ -1252,6 +1271,64 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
w2_bias,
|
||||
)
|
||||
|
||||
elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_BF16:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if w13_bias is not None:
|
||||
w13_bias = w13_bias.data.to(torch.float32)
|
||||
if w2_bias is not None:
|
||||
w2_bias = w2_bias.data.to(torch.float32)
|
||||
|
||||
e, n, k = w13_weight.shape
|
||||
|
||||
w13_weight.view(torch.uint8).copy_(
|
||||
w13_weight.data.view(torch.uint8)
|
||||
.view(e, n // 2, 2, k)
|
||||
.permute(0, 2, 1, 3)
|
||||
.contiguous()
|
||||
.view(e, n, k)
|
||||
)
|
||||
w13_weight_scale.data = (
|
||||
w13_weight_scale.data.view(e, n // 2, 2, -1)
|
||||
.permute(0, 2, 1, 3)
|
||||
.contiguous()
|
||||
.view(e, n, -1)
|
||||
)
|
||||
|
||||
w13_weight.data = w13_weight.data.view(torch.float4_e2m1fn_x2)
|
||||
w2_weight.data = w2_weight.data.view(torch.float4_e2m1fn_x2)
|
||||
|
||||
w13_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(w13_weight, 16, True)
|
||||
shuffled_w13_scale = rocm_aiter_ops.shuffle_scale_a16w4(
|
||||
w13_weight_scale.view(-1, w13_weight_scale.shape[-1]),
|
||||
num_experts,
|
||||
True,
|
||||
)
|
||||
|
||||
w2_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(w2_weight, 16, False)
|
||||
shuffled_w2_scale = rocm_aiter_ops.shuffle_scale_a16w4(
|
||||
w2_weight_scale.view(-1, w2_weight_scale.shape[-1]),
|
||||
num_experts,
|
||||
False,
|
||||
)
|
||||
|
||||
if w13_bias is not None:
|
||||
w13_bias = (
|
||||
w13_bias.data.view(-1, n // 2, 2)
|
||||
.permute(0, 2, 1)
|
||||
.contiguous()
|
||||
.view(-1, n)
|
||||
)
|
||||
|
||||
return (
|
||||
w13_weight,
|
||||
w2_weight,
|
||||
shuffled_w13_scale,
|
||||
shuffled_w2_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
|
||||
elif mxfp4_backend in TRITON_BACKENDS:
|
||||
from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig
|
||||
|
||||
@@ -1307,7 +1384,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported mxfp4_backend for Mxfp4MoEMethod: {mxfp4_backend}. "
|
||||
f"Expected TRTLLM or Triton backend."
|
||||
f"Expected TRTLLM, Triton, or AITER backend."
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -234,6 +234,39 @@ def mhc_pre(
|
||||
num_tokens = residual_flat.shape[0]
|
||||
fn_flat = fn
|
||||
|
||||
if current_platform.is_rocm():
|
||||
x = residual_flat.view(num_tokens, hc_mult * hidden_size).to(torch.float32)
|
||||
mixes = torch.matmul(x, fn_flat.t())
|
||||
sqrsum = x.square().sum(dim=-1, keepdim=True)
|
||||
mixes = mixes * torch.rsqrt(sqrsum / (hc_mult * hidden_size) + rms_eps)
|
||||
|
||||
pre_logits = mixes[:, :hc_mult] * hc_scale[0] + hc_base[:hc_mult]
|
||||
pre_mix = torch.sigmoid(pre_logits) + hc_pre_eps
|
||||
|
||||
post_logits = (
|
||||
mixes[:, hc_mult : 2 * hc_mult] * hc_scale[1]
|
||||
+ hc_base[hc_mult : 2 * hc_mult]
|
||||
)
|
||||
post_mix = torch.sigmoid(post_logits) * hc_post_mult_value
|
||||
|
||||
comb_logits = mixes[:, 2 * hc_mult :].view(
|
||||
num_tokens, hc_mult, hc_mult
|
||||
) * hc_scale[2] + hc_base[2 * hc_mult :].view(1, hc_mult, hc_mult)
|
||||
comb_mix = torch.softmax(comb_logits, dim=-1) + hc_sinkhorn_eps
|
||||
comb_mix = comb_mix / (comb_mix.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
|
||||
for _ in range(sinkhorn_repeat - 1):
|
||||
comb_mix = comb_mix / (comb_mix.sum(dim=-1, keepdim=True) + hc_sinkhorn_eps)
|
||||
comb_mix = comb_mix / (comb_mix.sum(dim=-2, keepdim=True) + hc_sinkhorn_eps)
|
||||
|
||||
layer_input = torch.sum(
|
||||
pre_mix.unsqueeze(-1) * residual_flat.to(torch.float32), dim=1
|
||||
).to(torch.bfloat16)
|
||||
return (
|
||||
post_mix.view(*outer_shape, hc_mult, 1),
|
||||
comb_mix.view(*outer_shape, hc_mult, hc_mult),
|
||||
layer_input.view(*outer_shape, hidden_size),
|
||||
)
|
||||
|
||||
# these number are from deepgemm kernel impl
|
||||
block_k = 64
|
||||
block_m = 64
|
||||
@@ -414,6 +447,14 @@ def mhc_post(
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
if current_platform.is_rocm():
|
||||
mixed_residual = torch.einsum(
|
||||
"...ij,...ih->...jh",
|
||||
comb_res_mix.to(torch.float32),
|
||||
residual.to(torch.float32),
|
||||
)
|
||||
post_term = post_layer_mix.to(torch.float32) * x.unsqueeze(-2).to(torch.float32)
|
||||
return (mixed_residual + post_term).to(residual.dtype)
|
||||
out = torch.empty_like(residual)
|
||||
mhc_post_tilelang(
|
||||
comb_res_mix,
|
||||
@@ -551,6 +592,49 @@ def hc_head_fuse_tilelang(
|
||||
T.pdl_trigger()
|
||||
|
||||
|
||||
def _hc_head_fused_reference(
|
||||
hs_flat: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
hidden_size: int,
|
||||
rms_eps: float,
|
||||
hc_eps: float,
|
||||
hc_mult: int,
|
||||
) -> None:
|
||||
"""Pure-PyTorch reference for `hc_head_fuse_tilelang`.
|
||||
|
||||
Used on platforms where the tilelang HIP/CUDA backend is not available
|
||||
(e.g. ROCm builds shipping a tilelang wheel without `target.build.tilelang_hip`).
|
||||
Mirrors the math of the tilelang kernel exactly:
|
||||
|
||||
x = hs_flat.flatten(-2, -1) # (T, hc_mult * H), fp32
|
||||
mixes = x @ fn.T # (T, hc_mult)
|
||||
rsqrt = 1 / sqrt(||x||^2 / (hc_mult * H) + rms_eps)
|
||||
pre[m] = sigmoid(mixes[m] * rsqrt * hc_scale[0] + hc_base[m]) + hc_eps
|
||||
out = sum_m pre[m] * hs_flat[:, m, :] # cast back to bf16
|
||||
|
||||
`out` is mutated in place to keep the same op contract
|
||||
(`mutates_args=["out"]`).
|
||||
"""
|
||||
num_tokens = hs_flat.shape[0]
|
||||
if num_tokens == 0:
|
||||
return
|
||||
x = hs_flat.reshape(num_tokens, hc_mult * hidden_size).to(torch.float32)
|
||||
# fn: (hc_mult, hc_mult * hidden_size) → mixes: (T, hc_mult)
|
||||
mixes = torch.matmul(x, fn.t())
|
||||
sqrsum = x.square().sum(dim=-1, keepdim=True)
|
||||
rsqrt = torch.rsqrt(sqrsum / (hc_mult * hidden_size) + rms_eps)
|
||||
# hc_scale has shape (1,); hc_base has shape (hc_mult,)
|
||||
pre_mix = torch.sigmoid(mixes * rsqrt * hc_scale[0] + hc_base) + hc_eps
|
||||
# weighted sum over the hc_mult channel dim
|
||||
result = torch.sum(pre_mix.unsqueeze(-1) * hs_flat.to(torch.float32), dim=1).to(
|
||||
out.dtype
|
||||
)
|
||||
out.copy_(result)
|
||||
|
||||
|
||||
def _hc_head_fused_kernel(
|
||||
hs_flat: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
@@ -563,8 +647,15 @@ def _hc_head_fused_kernel(
|
||||
hc_mult: int,
|
||||
) -> None:
|
||||
"""Fill pre-allocated `out` (T, H) in-place with the hc_head result."""
|
||||
if hs_flat.shape[0] > 0:
|
||||
hc_head_fuse_tilelang(
|
||||
if hs_flat.shape[0] == 0:
|
||||
return
|
||||
if current_platform.is_rocm():
|
||||
# tilelang ships only the CUDA codegen in upstream wheels, so the HIP
|
||||
# FFI target (`target.build.tilelang_hip`) is missing and the JIT call
|
||||
# would raise `ValueError: Cannot find global function ...`. Use a
|
||||
# numerically equivalent torch fallback instead. `mhc_pre` and
|
||||
# `mhc_post` already follow this same pattern above.
|
||||
_hc_head_fused_reference(
|
||||
hs_flat,
|
||||
fn,
|
||||
hc_scale,
|
||||
@@ -575,6 +666,18 @@ def _hc_head_fused_kernel(
|
||||
hc_eps,
|
||||
hc_mult,
|
||||
)
|
||||
return
|
||||
hc_head_fuse_tilelang(
|
||||
hs_flat,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
out,
|
||||
hidden_size,
|
||||
rms_eps,
|
||||
hc_eps,
|
||||
hc_mult,
|
||||
)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
|
||||
@@ -843,6 +843,15 @@ def w8a8_triton_block_scaled_mm(
|
||||
assert len(block_size) == 2
|
||||
block_n, block_k = block_size[0], block_size[1]
|
||||
|
||||
# Triton cannot currently bind E8M0 scale tensors directly. On ROCm,
|
||||
# DeepSeek-V4 checkpoints store block scales in exponent-only E8M0 format,
|
||||
# so decode them to fp32 before launching the kernel.
|
||||
if current_platform.is_rocm():
|
||||
if As.dtype == torch.float8_e8m0fnu:
|
||||
As = _upcast_e8m0_to_fp32(As).contiguous()
|
||||
if Bs.dtype == torch.float8_e8m0fnu:
|
||||
Bs = _upcast_e8m0_to_fp32(Bs).contiguous()
|
||||
|
||||
assert A.shape[-1] == B.shape[-1]
|
||||
assert A.shape[:-1] == As.shape[:-1] and A.is_contiguous()
|
||||
assert triton.cdiv(A.shape[-1], block_k) == As.shape[-1]
|
||||
|
||||
@@ -499,13 +499,31 @@ class SparseAttnIndexer(CustomOp):
|
||||
k: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
):
|
||||
assert not self.skip_k_cache_insert, (
|
||||
"AMD platform doesn't support skip cache insert yet"
|
||||
)
|
||||
assert not self.use_fp4_cache, "AMD platform doesn't support fp4 cache yet"
|
||||
assert isinstance(q_quant, torch.Tensor), (
|
||||
"AMD sparse_attn_indexer expects a single FP8 q_quant tensor"
|
||||
)
|
||||
if self.skip_k_cache_insert or not rocm_aiter_ops.is_enabled():
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
rocm_aiter_sparse_attn_indexer_native,
|
||||
)
|
||||
|
||||
return rocm_aiter_sparse_attn_indexer_native(
|
||||
hidden_states,
|
||||
_encode_layer_name(self.k_cache.prefix),
|
||||
self.k_cache.kv_cache,
|
||||
q_quant,
|
||||
k,
|
||||
weights,
|
||||
self.quant_block_size,
|
||||
self.scale_fmt,
|
||||
self.topk_tokens,
|
||||
self.head_dim,
|
||||
self.max_model_len,
|
||||
self.max_total_seq_len,
|
||||
self.topk_indices_buffer,
|
||||
skip_k_cache_insert=self.skip_k_cache_insert,
|
||||
)
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
return torch.ops.vllm.rocm_aiter_sparse_attn_indexer(
|
||||
hidden_states,
|
||||
@@ -522,8 +540,4 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.max_total_seq_len,
|
||||
self.topk_indices_buffer,
|
||||
)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"Sparse attention indexer ROCm custom op requires ROCm "
|
||||
"Aiter ops to be enabled."
|
||||
)
|
||||
raise RuntimeError("Sparse attention indexer ROCm path could not be selected.")
|
||||
|
||||
@@ -1245,7 +1245,12 @@ class DeepseekV4Model(nn.Module):
|
||||
# DeepseekV4MultiHeadLatentAttentionWrapper.attn_gemm_parallel_execute
|
||||
# (compressor kv_score, indexer.weights_proj, indexer.compressor
|
||||
# kv_score). fused_wqa_wkv stays on the default stream.
|
||||
aux_stream_list = [torch.cuda.Stream() for _ in range(3)]
|
||||
# Disable them on ROCm because of hang issues.
|
||||
aux_stream_list = (
|
||||
None
|
||||
if current_platform.is_rocm()
|
||||
else [torch.cuda.Stream() for _ in range(3)]
|
||||
)
|
||||
|
||||
self.device = current_platform.device_type
|
||||
# Reserved topk indices buffer for all Indexer layers to reuse.
|
||||
|
||||
@@ -167,8 +167,12 @@ class DeepSeekV4MultiTokenPredictor(nn.Module):
|
||||
)
|
||||
|
||||
# Three aux streams shared across all MTP layers, mirroring
|
||||
# DeepseekV4Model.
|
||||
aux_stream_list = [torch.cuda.Stream() for _ in range(3)]
|
||||
# DeepseekV4Model. ROCm runs the same work serially for now.
|
||||
aux_stream_list = (
|
||||
None
|
||||
if current_platform.is_rocm()
|
||||
else [torch.cuda.Stream() for _ in range(3)]
|
||||
)
|
||||
|
||||
# to map the exact layer index from weights
|
||||
self.layers = torch.nn.ModuleDict(
|
||||
|
||||
Reference in New Issue
Block a user