[ROCm][DSv4] Functional fixes for DeepSeek V4 on MI300X/MI325X (#45681)

Signed-off-by: ganyi <ygan@amd.com>
Signed-off-by: Markus Hartikainen <markus.hartikainen@amd.com>
Signed-off-by: Tuukka Sarvi <tuukka.sarvi@amd.com>
Co-authored-by: ganyi <ygan@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Markus Hartikainen <markus.hartikainen@amd.com>
Co-authored-by: Jin Tao <jintao12@amd.com>
This commit is contained in:
Tuukka Sarvi
2026-06-18 12:21:14 +00:00
committed by GitHub
co-authored by ganyi Cursor Markus Hartikainen Jin Tao
parent 8d4f54966c
commit afdcbd5d39
8 changed files with 545 additions and 52 deletions
@@ -18,7 +18,7 @@
* ROPE_DIM = 64 (RoPE applied to dims [NOPE_DIM, HEAD_DIM))
* NOPE_DIM = 448
* QUANT_BLOCK = 64 (UE8M0 FP8 quant block)
* FP8_MAX = 448.0f
* FP8_MAX = 224.0f on ROCm FNUZ / 448.0f on OCP
* is_neox=false (GPT-J interleaved pairs)
* cos_sin_cache layout [max_pos, rope_dim] = cos || sin (cos first, sin
* second along last dim; each half is rope_dim/2 = 32 values)
@@ -61,10 +61,11 @@
#ifdef USE_ROCM
// ROCm-compatible FP8 conversion helpers
__device__ __forceinline__ uint8_t rocm_cvt_float_to_fp8_e4m3(float val) {
#if defined(HIP_FP8_TYPE_OCP)
__hip_fp8_e4m3 fp8_val(val);
#else
// gfx942 uses FNUZ FP8; other ROCm targets use OCP E4M3.
#if defined(__gfx942__)
__hip_fp8_e4m3_fnuz fp8_val(val);
#else
__hip_fp8_e4m3 fp8_val(val);
#endif
return reinterpret_cast<uint8_t&>(fp8_val);
}
@@ -90,7 +91,13 @@ constexpr int kQuantBlock = 64;
constexpr int kNumQuantBlocks = kNopeDim / kQuantBlock; // 7
constexpr int kScaleBytesPerToken = kNumQuantBlocks + 1; // 8 (7 real + 1 pad)
constexpr int kTokenDataBytes = kNopeDim + kRopeDim * 2; // 448 + 128 = 576
// FNUZ on gfx942 / OCP elsewhere. FNUZ uses 224.0 (not the dtype's raw
// 240.0) to match the rest of vLLM's FNUZ pipeline.
#if defined(USE_ROCM) && defined(__gfx942__)
constexpr float kFp8Max = 224.0f;
#else
constexpr float kFp8Max = 448.0f;
#endif
#ifndef USE_ROCM
// When num_tokens is less than this threshold,