[ROCm][DSv4] Functional fixes for DeepSeek V4 on MI300X/MI325X (#45681)
Signed-off-by: ganyi <ygan@amd.com> Signed-off-by: Markus Hartikainen <markus.hartikainen@amd.com> Signed-off-by: Tuukka Sarvi <tuukka.sarvi@amd.com> Co-authored-by: ganyi <ygan@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Markus Hartikainen <markus.hartikainen@amd.com> Co-authored-by: Jin Tao <jintao12@amd.com>
This commit is contained in:
co-authored by
ganyi
Cursor
Markus Hartikainen
Jin Tao
parent
8d4f54966c
commit
afdcbd5d39
@@ -18,7 +18,7 @@
|
||||
* ROPE_DIM = 64 (RoPE applied to dims [NOPE_DIM, HEAD_DIM))
|
||||
* NOPE_DIM = 448
|
||||
* QUANT_BLOCK = 64 (UE8M0 FP8 quant block)
|
||||
* FP8_MAX = 448.0f
|
||||
* FP8_MAX = 224.0f on ROCm FNUZ / 448.0f on OCP
|
||||
* is_neox=false (GPT-J interleaved pairs)
|
||||
* cos_sin_cache layout [max_pos, rope_dim] = cos || sin (cos first, sin
|
||||
* second along last dim; each half is rope_dim/2 = 32 values)
|
||||
@@ -61,10 +61,11 @@
|
||||
#ifdef USE_ROCM
|
||||
// ROCm-compatible FP8 conversion helpers
|
||||
__device__ __forceinline__ uint8_t rocm_cvt_float_to_fp8_e4m3(float val) {
|
||||
#if defined(HIP_FP8_TYPE_OCP)
|
||||
__hip_fp8_e4m3 fp8_val(val);
|
||||
#else
|
||||
// gfx942 uses FNUZ FP8; other ROCm targets use OCP E4M3.
|
||||
#if defined(__gfx942__)
|
||||
__hip_fp8_e4m3_fnuz fp8_val(val);
|
||||
#else
|
||||
__hip_fp8_e4m3 fp8_val(val);
|
||||
#endif
|
||||
return reinterpret_cast<uint8_t&>(fp8_val);
|
||||
}
|
||||
@@ -90,7 +91,13 @@ constexpr int kQuantBlock = 64;
|
||||
constexpr int kNumQuantBlocks = kNopeDim / kQuantBlock; // 7
|
||||
constexpr int kScaleBytesPerToken = kNumQuantBlocks + 1; // 8 (7 real + 1 pad)
|
||||
constexpr int kTokenDataBytes = kNopeDim + kRopeDim * 2; // 448 + 128 = 576
|
||||
// FNUZ on gfx942 / OCP elsewhere. FNUZ uses 224.0 (not the dtype's raw
|
||||
// 240.0) to match the rest of vLLM's FNUZ pipeline.
|
||||
#if defined(USE_ROCM) && defined(__gfx942__)
|
||||
constexpr float kFp8Max = 224.0f;
|
||||
#else
|
||||
constexpr float kFp8Max = 448.0f;
|
||||
#endif
|
||||
|
||||
#ifndef USE_ROCM
|
||||
// When num_tokens is less than this threshold,
|
||||
|
||||
Reference in New Issue
Block a user