[Bugfix]fix output Nan/Inf in marlin if dtype=float16 (#33972)
Signed-off-by: IriKa Qiu <qiujie.jq@gmail.com>
This commit is contained in:
@@ -189,10 +189,7 @@ __device__ __forceinline__ void cp_async_wait<0>() {
|
||||
}
|
||||
|
||||
__device__ __forceinline__ float clip(float v, float mmin, float mmax) {
|
||||
#if __CUDACC_VER_MAJOR__ >= 11 && __CUDA_ARCH__ >= 800
|
||||
return fminf(mmax, fmaxf(v, mmin));
|
||||
#else
|
||||
#endif
|
||||
}
|
||||
|
||||
__device__ __forceinline__ __nv_bfloat16 clip(__nv_bfloat16 v,
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
const int4 *__restrict__ b_bias_ptr, \
|
||||
const float *__restrict__ a_scales_ptr, \
|
||||
const int4 *__restrict__ scales_ptr, \
|
||||
const uint16_t *__restrict__ global_scale_ptr, \
|
||||
const float *__restrict__ global_scale_ptr, \
|
||||
const int4 *__restrict__ zp_ptr, const int *__restrict__ g_idx, \
|
||||
int num_groups, int prob_m, int prob_n, int prob_k, int lda, int *locks, \
|
||||
bool has_bias, bool use_atomic_add, bool use_fp32_reduce, \
|
||||
|
||||
@@ -57,7 +57,7 @@ torch::Tensor marlin_gemm(
|
||||
int64_t size_k, bool is_k_full, bool use_atomic_add, bool use_fp32_reduce,
|
||||
bool is_zp_float) {
|
||||
TORCH_CHECK_NOT_IMPLEMENTED(false,
|
||||
"marlin_gemm(..) requires CUDA_ARCH >= 8.0");
|
||||
"marlin_gemm(..) requires CUDA_ARCH >= 7.5");
|
||||
return torch::empty({1, 1});
|
||||
}
|
||||
|
||||
@@ -356,7 +356,7 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
|
||||
const int4* bias_ptr = (const int4*)b_bias;
|
||||
const float* a_s_ptr = (const float*)a_s;
|
||||
const int4* b_s_ptr = (const int4*)b_s;
|
||||
const uint16_t* g_s_ptr = (const uint16_t*)g_s;
|
||||
const float* g_s_ptr = (const float*)g_s;
|
||||
|
||||
const int4* zp_ptr = (const int4*)zp;
|
||||
const int* g_idx_ptr = (const int*)g_idx;
|
||||
@@ -751,7 +751,7 @@ torch::Tensor marlin_gemm(
|
||||
TORCH_CHECK(b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn,
|
||||
"global_scale can only be used for nvfp4 format.");
|
||||
} else {
|
||||
global_scale = torch::empty({0}, options);
|
||||
global_scale = torch::empty({0}, options_fp32);
|
||||
TORCH_CHECK(!(b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn),
|
||||
"the global_scale parameter must be passed for nvfp4 format.");
|
||||
}
|
||||
@@ -832,8 +832,8 @@ torch::Tensor marlin_gemm(
|
||||
|
||||
TORCH_CHECK(a_scales.scalar_type() == at::ScalarType::Float,
|
||||
"scalar type of a_scales must be float");
|
||||
TORCH_CHECK(global_scale.scalar_type() == c.scalar_type(),
|
||||
"scalar type of global_scale must be the same with c");
|
||||
TORCH_CHECK(global_scale.scalar_type() == at::ScalarType::Float,
|
||||
"scalar type of global_scale must be float");
|
||||
if (a_type.size_bits() == 16) {
|
||||
TORCH_CHECK(
|
||||
a.scalar_type() == c.scalar_type(),
|
||||
|
||||
@@ -251,8 +251,8 @@ __global__ void Marlin(
|
||||
const float* __restrict__ a_scales_ptr,
|
||||
// fp16 quantization scales. shape (k/groupsize, n)
|
||||
const int4* __restrict__ scales_ptr,
|
||||
// fp16 global scale (for nvfp4// only)
|
||||
const uint16_t* __restrict__ global_scale_ptr,
|
||||
// float global scale (for nvfp4// only)
|
||||
const float* __restrict__ global_scale_ptr,
|
||||
// 4bit packed zero-points of shape
|
||||
// (k/groupsize, n/pack_factor)
|
||||
const int4* __restrict__ zp_ptr,
|
||||
@@ -292,7 +292,13 @@ __global__ void Marlin(
|
||||
#endif
|
||||
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ == 750
|
||||
constexpr bool use_fp16_accum = a_type_id == vllm::kFloat16.id();
|
||||
constexpr auto num_bits = vllm::ScalarType::from_id(b_type_id).size_bits();
|
||||
// Disable use_fp16_accum for NVFP4 and cases when group_size == -1 &&
|
||||
// num_bits == 4
|
||||
constexpr bool use_fp16_accum =
|
||||
a_type_id == vllm::kFloat16.id() &&
|
||||
(!(b_type_id == vllm::kFE2M1f.id() && s_type_id == vllm::kFE4M3fn.id()) &&
|
||||
!(group_blocks == -1 && num_bits == 4));
|
||||
#else
|
||||
constexpr bool use_fp16_accum = false;
|
||||
#endif
|
||||
@@ -342,11 +348,10 @@ __global__ void Marlin(
|
||||
has_zp && !is_zp_float && !std::is_same<scalar_t, nv_bfloat16>::value ||
|
||||
has_zp && !is_zp_float && !(b_type == vllm::kU8);
|
||||
|
||||
c_scalar_t2 global_scale;
|
||||
float global_scale_f32 = 1.0f;
|
||||
|
||||
if constexpr (b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn) {
|
||||
uint16_t val = global_scale_ptr[0];
|
||||
global_scale = Cdtype::num2num2(*reinterpret_cast<c_scalar_t*>(&val));
|
||||
global_scale_f32 = global_scale_ptr[0];
|
||||
}
|
||||
|
||||
constexpr bool has_act_order = group_blocks == 0;
|
||||
@@ -1644,6 +1649,10 @@ __global__ void Marlin(
|
||||
// We first reorder in shared memory to guarantee the most efficient final
|
||||
// global write patterns
|
||||
auto write = [&](int idx, float c0, float c1, FragS& s, FragS& b_bias) {
|
||||
if constexpr (b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn) {
|
||||
c0 *= global_scale_f32;
|
||||
c1 *= global_scale_f32;
|
||||
}
|
||||
c_scalar_t2 res =
|
||||
Cdtype::nums2num2(Cdtype::float2num(c0), Cdtype::float2num(c1));
|
||||
|
||||
@@ -1659,10 +1668,6 @@ __global__ void Marlin(
|
||||
}
|
||||
res = __hmul2(res, tmp_scale);
|
||||
}
|
||||
|
||||
if constexpr (b_type == vllm::kFE2M1f && s_type == vllm::kFE4M3fn) {
|
||||
res = __hmul2(res, global_scale);
|
||||
}
|
||||
if (has_bias && last) {
|
||||
c_scalar_t2 tmp_bias = b_bias[0];
|
||||
if constexpr (m_block_size_8) {
|
||||
|
||||
Reference in New Issue
Block a user