[ROCm][DSv4] Functional fixes for DeepSeek V4 on MI300X/MI325X (#45681)
Signed-off-by: ganyi <ygan@amd.com> Signed-off-by: Markus Hartikainen <markus.hartikainen@amd.com> Signed-off-by: Tuukka Sarvi <tuukka.sarvi@amd.com> Co-authored-by: ganyi <ygan@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Markus Hartikainen <markus.hartikainen@amd.com> Co-authored-by: Jin Tao <jintao12@amd.com>
This commit is contained in:
co-authored by
ganyi
Cursor
Markus Hartikainen
Jin Tao
parent
8d4f54966c
commit
afdcbd5d39
@@ -14,6 +14,7 @@ from vllm.models.deepseek_v4.sparse_mla import (
|
||||
DeepseekV4FlashMLAMetadata,
|
||||
DeepseekV4FlashMLAMetadataBuilder,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backend import (
|
||||
CommonAttentionMetadata,
|
||||
@@ -796,6 +797,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
assert attn_metadata is not None
|
||||
assert compressed_k_cache is not None
|
||||
block_table = attn_metadata.block_table[num_decodes:]
|
||||
# compressed_k_cache is OCP on every platform (Triton encoder).
|
||||
dequantize_and_gather_k_cache(
|
||||
kv[:chunk_size],
|
||||
compressed_k_cache,
|
||||
@@ -804,6 +806,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
block_table=block_table[chunk_start:chunk_end],
|
||||
block_size=attn_metadata.block_size // self.compress_ratio,
|
||||
offset=0,
|
||||
use_fnuz=False,
|
||||
)
|
||||
|
||||
swa_block_table = swa_metadata.block_table[num_decodes:]
|
||||
@@ -815,6 +818,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
block_table=swa_block_table[chunk_start:chunk_end],
|
||||
block_size=swa_metadata.block_size,
|
||||
offset=N,
|
||||
use_fnuz=current_platform.is_fp8_fnuz(),
|
||||
)
|
||||
|
||||
query_start = (
|
||||
|
||||
@@ -16,6 +16,10 @@ preparation.
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
|
||||
@@ -39,6 +43,7 @@ def quantize_and_insert_k_kernel(
|
||||
block_stride: tl.constexpr, # total bytes per block (padded)
|
||||
fp8_max: tl.constexpr,
|
||||
n_quant_blocks: tl.constexpr, # 8 (7 real + 1 padding)
|
||||
use_fnuz: tl.constexpr = False,
|
||||
):
|
||||
"""
|
||||
Quantize K tensor and insert into paged K cache.
|
||||
@@ -49,6 +54,9 @@ def quantize_and_insert_k_kernel(
|
||||
- [64*576 + 64*8, block_stride): Padding
|
||||
|
||||
One program per token.
|
||||
|
||||
``use_fnuz=True`` selects FNUZ (``tl.float8e4b8``); default OCP
|
||||
(``tl.float8e4nv``) matches every production caller.
|
||||
"""
|
||||
pid = tl.program_id(0)
|
||||
|
||||
@@ -112,8 +120,11 @@ def quantize_and_insert_k_kernel(
|
||||
x_scaled = x / scale
|
||||
x_clamped = tl.clamp(x_scaled, -fp8_max, fp8_max)
|
||||
|
||||
# Convert to fp8, then bitcast to uint8 for storage
|
||||
x_fp8 = x_clamped.to(tl.float8e4nv)
|
||||
# Convert to fp8 (FNUZ on gfx942, OCP elsewhere), then bitcast to uint8.
|
||||
if use_fnuz:
|
||||
x_fp8 = x_clamped.to(tl.float8e4b8)
|
||||
else:
|
||||
x_fp8 = x_clamped.to(tl.float8e4nv)
|
||||
x_uint8 = x_fp8.to(tl.uint8, bitcast=True)
|
||||
|
||||
# Store as uint8 (1 byte each)
|
||||
@@ -145,6 +156,7 @@ def quantize_and_insert_k_cache(
|
||||
slot_mapping: torch.Tensor, # [num_tokens] int64
|
||||
block_size: int = 64,
|
||||
is_ue8m0: bool = True,
|
||||
use_fnuz: bool = False,
|
||||
):
|
||||
"""
|
||||
Quantize K tensor and insert into paged K cache.
|
||||
@@ -155,6 +167,10 @@ def quantize_and_insert_k_cache(
|
||||
- Next 64 * 8 = 512 bytes: Scales
|
||||
- Each token: 8 bytes (uint8 scales, 7 real + 1 padding)
|
||||
- Padded to multiple of 576
|
||||
|
||||
``use_fnuz=True`` selects FNUZ E4M3 cache encoding and is only valid on
|
||||
platforms whose FP8 format is FNUZ. ``use_fnuz=False`` selects OCP E4M3,
|
||||
which is used by OCP-encoded caches even on gfx942.
|
||||
"""
|
||||
assert k.dim() == 2 and k.shape[1] == 512, (
|
||||
f"K must be [num_tokens, 512], got {k.shape}"
|
||||
@@ -171,7 +187,12 @@ def quantize_and_insert_k_cache(
|
||||
TOKEN_BF16_DIM = 64
|
||||
TOKEN_SCALE_DIM = 8
|
||||
QUANT_BLOCK_SIZE = 64
|
||||
FP8_MAX = 448.0
|
||||
if use_fnuz:
|
||||
if not current_platform.is_fp8_fnuz():
|
||||
raise ValueError("use_fnuz=True requires a platform using FNUZ FP8")
|
||||
_, FP8_MAX = get_fp8_min_max()
|
||||
else:
|
||||
FP8_MAX = torch.finfo(torch.float8_e4m3fn).max
|
||||
TOKEN_DATA_SIZE = TOKEN_FP8_DIM + TOKEN_BF16_DIM * 2
|
||||
|
||||
grid = (num_tokens,)
|
||||
@@ -191,6 +212,7 @@ def quantize_and_insert_k_cache(
|
||||
block_stride=block_stride,
|
||||
fp8_max=FP8_MAX,
|
||||
n_quant_blocks=8,
|
||||
use_fnuz=use_fnuz,
|
||||
)
|
||||
|
||||
|
||||
@@ -216,6 +238,7 @@ def _dequantize_and_gather_k_kernel(
|
||||
output_dim: tl.constexpr, # 512
|
||||
fp8_max: tl.constexpr,
|
||||
n_quant_blocks: tl.constexpr, # 7 real blocks
|
||||
use_fnuz: tl.constexpr = False,
|
||||
):
|
||||
batch_idx = tl.program_id(0)
|
||||
worker_id = tl.program_id(1)
|
||||
@@ -273,8 +296,11 @@ def _dequantize_and_gather_k_kernel(
|
||||
# Load quantized fp8 values (stored as uint8)
|
||||
x_uint8 = tl.load(token_fp8_ptr + offsets, mask=mask, other=0)
|
||||
|
||||
# Bitcast uint8 back to fp8
|
||||
x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True)
|
||||
# Bitcast uint8 back to fp8 (FNUZ on gfx942, OCP elsewhere).
|
||||
if use_fnuz:
|
||||
x_fp8 = x_uint8.to(tl.float8e4b8, bitcast=True)
|
||||
else:
|
||||
x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True)
|
||||
|
||||
# Convert fp8 to float32 for computation
|
||||
x_float = x_fp8.to(tl.float32)
|
||||
@@ -317,6 +343,7 @@ def dequantize_and_gather_k_cache_triton(
|
||||
block_table: torch.Tensor,
|
||||
block_size: int,
|
||||
offset: int,
|
||||
use_fnuz: bool = False,
|
||||
) -> None:
|
||||
TOKEN_FP8_DIM = 448
|
||||
TOKEN_BF16_DIM = 64
|
||||
@@ -347,6 +374,7 @@ def dequantize_and_gather_k_cache_triton(
|
||||
output_dim=512,
|
||||
fp8_max=FP8_MAX,
|
||||
n_quant_blocks=7,
|
||||
use_fnuz=use_fnuz,
|
||||
)
|
||||
|
||||
|
||||
@@ -363,7 +391,15 @@ def dequantize_and_gather_k_cache(
|
||||
block_table: torch.Tensor,
|
||||
block_size: int,
|
||||
offset: int,
|
||||
use_fnuz: bool = False,
|
||||
) -> None:
|
||||
"""Dequantize and gather a paged DSv4 K cache.
|
||||
|
||||
``use_fnuz`` MUST match the encoder of the specific cache being read:
|
||||
``False`` for ``compressed_k_cache`` (Triton encoder is OCP everywhere),
|
||||
``current_platform.is_fp8_fnuz()`` for ``swa_k_cache`` (C++ encoder
|
||||
writes FNUZ on gfx942 and OCP on gfx950).
|
||||
"""
|
||||
if has_cutedsl():
|
||||
# lazily import, otherwise some tests fail due to CUDA driver init failure.
|
||||
from vllm.models.deepseek_v4.nvidia.ops.dequant_gather_k_cutedsl import (
|
||||
@@ -376,7 +412,14 @@ def dequantize_and_gather_k_cache(
|
||||
return
|
||||
|
||||
dequantize_and_gather_k_cache_triton(
|
||||
out, k_cache, seq_lens, gather_lens, block_table, block_size, offset
|
||||
out,
|
||||
k_cache,
|
||||
seq_lens,
|
||||
gather_lens,
|
||||
block_table,
|
||||
block_size,
|
||||
offset,
|
||||
use_fnuz=use_fnuz,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.models.deepseek_v4.common.ops import fused_inv_rope_fp8_quant
|
||||
from vllm.models.deepseek_v4.common.ops.fused_inv_rope_fp8_quant import (
|
||||
fused_inv_rope_fp8_quant,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import fp8_einsum
|
||||
|
||||
|
||||
Reference in New Issue
Block a user