[ROCm][DSv4] Functional fixes for DeepSeek V4 on MI300X/MI325X (#45681)

Signed-off-by: ganyi <ygan@amd.com>
Signed-off-by: Markus Hartikainen <markus.hartikainen@amd.com>
Signed-off-by: Tuukka Sarvi <tuukka.sarvi@amd.com>
Co-authored-by: ganyi <ygan@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Markus Hartikainen <markus.hartikainen@amd.com>
Co-authored-by: Jin Tao <jintao12@amd.com>
This commit is contained in:
Tuukka Sarvi
2026-06-18 12:21:14 +00:00
committed by GitHub
co-authored by ganyi Cursor Markus Hartikainen Jin Tao
parent 8d4f54966c
commit afdcbd5d39
8 changed files with 545 additions and 52 deletions
+4
View File
@@ -14,6 +14,7 @@ from vllm.models.deepseek_v4.sparse_mla import (
DeepseekV4FlashMLAMetadata,
DeepseekV4FlashMLAMetadataBuilder,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backend import (
CommonAttentionMetadata,
@@ -796,6 +797,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
assert attn_metadata is not None
assert compressed_k_cache is not None
block_table = attn_metadata.block_table[num_decodes:]
# compressed_k_cache is OCP on every platform (Triton encoder).
dequantize_and_gather_k_cache(
kv[:chunk_size],
compressed_k_cache,
@@ -804,6 +806,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
block_table=block_table[chunk_start:chunk_end],
block_size=attn_metadata.block_size // self.compress_ratio,
offset=0,
use_fnuz=False,
)
swa_block_table = swa_metadata.block_table[num_decodes:]
@@ -815,6 +818,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
block_table=swa_block_table[chunk_start:chunk_end],
block_size=swa_metadata.block_size,
offset=N,
use_fnuz=current_platform.is_fp8_fnuz(),
)
query_start = (
@@ -16,6 +16,10 @@ preparation.
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.import_utils import has_cutedsl
@@ -39,6 +43,7 @@ def quantize_and_insert_k_kernel(
block_stride: tl.constexpr, # total bytes per block (padded)
fp8_max: tl.constexpr,
n_quant_blocks: tl.constexpr, # 8 (7 real + 1 padding)
use_fnuz: tl.constexpr = False,
):
"""
Quantize K tensor and insert into paged K cache.
@@ -49,6 +54,9 @@ def quantize_and_insert_k_kernel(
- [64*576 + 64*8, block_stride): Padding
One program per token.
``use_fnuz=True`` selects FNUZ (``tl.float8e4b8``); default OCP
(``tl.float8e4nv``) matches every production caller.
"""
pid = tl.program_id(0)
@@ -112,8 +120,11 @@ def quantize_and_insert_k_kernel(
x_scaled = x / scale
x_clamped = tl.clamp(x_scaled, -fp8_max, fp8_max)
# Convert to fp8, then bitcast to uint8 for storage
x_fp8 = x_clamped.to(tl.float8e4nv)
# Convert to fp8 (FNUZ on gfx942, OCP elsewhere), then bitcast to uint8.
if use_fnuz:
x_fp8 = x_clamped.to(tl.float8e4b8)
else:
x_fp8 = x_clamped.to(tl.float8e4nv)
x_uint8 = x_fp8.to(tl.uint8, bitcast=True)
# Store as uint8 (1 byte each)
@@ -145,6 +156,7 @@ def quantize_and_insert_k_cache(
slot_mapping: torch.Tensor, # [num_tokens] int64
block_size: int = 64,
is_ue8m0: bool = True,
use_fnuz: bool = False,
):
"""
Quantize K tensor and insert into paged K cache.
@@ -155,6 +167,10 @@ def quantize_and_insert_k_cache(
- Next 64 * 8 = 512 bytes: Scales
- Each token: 8 bytes (uint8 scales, 7 real + 1 padding)
- Padded to multiple of 576
``use_fnuz=True`` selects FNUZ E4M3 cache encoding and is only valid on
platforms whose FP8 format is FNUZ. ``use_fnuz=False`` selects OCP E4M3,
which is used by OCP-encoded caches even on gfx942.
"""
assert k.dim() == 2 and k.shape[1] == 512, (
f"K must be [num_tokens, 512], got {k.shape}"
@@ -171,7 +187,12 @@ def quantize_and_insert_k_cache(
TOKEN_BF16_DIM = 64
TOKEN_SCALE_DIM = 8
QUANT_BLOCK_SIZE = 64
FP8_MAX = 448.0
if use_fnuz:
if not current_platform.is_fp8_fnuz():
raise ValueError("use_fnuz=True requires a platform using FNUZ FP8")
_, FP8_MAX = get_fp8_min_max()
else:
FP8_MAX = torch.finfo(torch.float8_e4m3fn).max
TOKEN_DATA_SIZE = TOKEN_FP8_DIM + TOKEN_BF16_DIM * 2
grid = (num_tokens,)
@@ -191,6 +212,7 @@ def quantize_and_insert_k_cache(
block_stride=block_stride,
fp8_max=FP8_MAX,
n_quant_blocks=8,
use_fnuz=use_fnuz,
)
@@ -216,6 +238,7 @@ def _dequantize_and_gather_k_kernel(
output_dim: tl.constexpr, # 512
fp8_max: tl.constexpr,
n_quant_blocks: tl.constexpr, # 7 real blocks
use_fnuz: tl.constexpr = False,
):
batch_idx = tl.program_id(0)
worker_id = tl.program_id(1)
@@ -273,8 +296,11 @@ def _dequantize_and_gather_k_kernel(
# Load quantized fp8 values (stored as uint8)
x_uint8 = tl.load(token_fp8_ptr + offsets, mask=mask, other=0)
# Bitcast uint8 back to fp8
x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True)
# Bitcast uint8 back to fp8 (FNUZ on gfx942, OCP elsewhere).
if use_fnuz:
x_fp8 = x_uint8.to(tl.float8e4b8, bitcast=True)
else:
x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True)
# Convert fp8 to float32 for computation
x_float = x_fp8.to(tl.float32)
@@ -317,6 +343,7 @@ def dequantize_and_gather_k_cache_triton(
block_table: torch.Tensor,
block_size: int,
offset: int,
use_fnuz: bool = False,
) -> None:
TOKEN_FP8_DIM = 448
TOKEN_BF16_DIM = 64
@@ -347,6 +374,7 @@ def dequantize_and_gather_k_cache_triton(
output_dim=512,
fp8_max=FP8_MAX,
n_quant_blocks=7,
use_fnuz=use_fnuz,
)
@@ -363,7 +391,15 @@ def dequantize_and_gather_k_cache(
block_table: torch.Tensor,
block_size: int,
offset: int,
use_fnuz: bool = False,
) -> None:
"""Dequantize and gather a paged DSv4 K cache.
``use_fnuz`` MUST match the encoder of the specific cache being read:
``False`` for ``compressed_k_cache`` (Triton encoder is OCP everywhere),
``current_platform.is_fp8_fnuz()`` for ``swa_k_cache`` (C++ encoder
writes FNUZ on gfx942 and OCP on gfx950).
"""
if has_cutedsl():
# lazily import, otherwise some tests fail due to CUDA driver init failure.
from vllm.models.deepseek_v4.nvidia.ops.dequant_gather_k_cutedsl import (
@@ -376,7 +412,14 @@ def dequantize_and_gather_k_cache(
return
dequantize_and_gather_k_cache_triton(
out, k_cache, seq_lens, gather_lens, block_table, block_size, offset
out,
k_cache,
seq_lens,
gather_lens,
block_table,
block_size,
offset,
use_fnuz=use_fnuz,
)
+3 -1
View File
@@ -3,7 +3,9 @@
import torch
import torch.nn as nn
from vllm.models.deepseek_v4.common.ops import fused_inv_rope_fp8_quant
from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import (
fused_inv_rope_fp8_quant,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import fp8_einsum