[Build] Fix CUDA arch build coverage gaps (#45277)

Signed-off-by: Shengqi Chen <harry-chen@outlook.com>
Co-authored-by: Xin Li <xinli-sw@users.noreply.github.com>
Co-authored-by: ShawRong <ShawRong@users.noreply.github.com>
Co-authored-by: Change72 <Change72@users.noreply.github.com>
This commit is contained in:
Shengqi Chen
2026-06-13 22:09:20 -07:00
committed by GitHub
co-authored by Xin Li ShawRong Change72
parent 54bbf51668
commit 78e7293bb1
14 changed files with 240 additions and 132 deletions
@@ -27,15 +27,24 @@
#include <torch/csrc/stable/tensor.h>
#include "libtorch_stable/torch_utils.h"
#include "libtorch_stable/dispatch_utils.h"
#include "libtorch_stable/cutlass_extensions/common.hpp"
#include "../../cuda_vec_utils.cuh"
#include "cuda_utils.h"
#include "nvfp4_utils.cuh"
#if defined(CUDART_VERSION) && CUDART_VERSION >= 12090
#define VLLM_MXFP4_EXPERTS_QUANT_SUPPORTED 1
static_assert(CVT_FP4_ELTS_PER_THREAD == 16,
"MXFP4 experts quant requires PACK16 mode (CUDA >= 12.9)");
#else
#define VLLM_MXFP4_EXPERTS_QUANT_SUPPORTED 0
#endif
#include "libtorch_stable/launch_bounds_utils.h"
#if VLLM_MXFP4_EXPERTS_QUANT_SUPPORTED
namespace vllm {
// MXFP4 block size constants
@@ -104,7 +113,7 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
&input_offset_by_experts[chunk_start + 12]));
local_offsets[16] = __ldca(&input_offset_by_experts[chunk_start + 16]);
#pragma unroll
#pragma unroll
for (int i = 0; i < 16; i++) {
if (rowIdx >= local_offsets[i] && rowIdx < local_offsets[i + 1]) {
rowIdx_in_expert = rowIdx - local_offsets[i];
@@ -309,14 +318,14 @@ void mxfp4_quant_impl(void* output, void* output_scale, void* input,
} // namespace vllm
/*Quantization entry for mxfp4 experts quantization*/
#define CHECK_TH_CUDA(x, m) \
STD_TORCH_CHECK(x.is_cuda(), m, "must be a CUDA tensor")
#define CHECK_CONTIGUOUS(x, m) \
STD_TORCH_CHECK(x.is_contiguous(), m, "must be contiguous")
#define CHECK_INPUT(x, m) \
CHECK_TH_CUDA(x, m); \
CHECK_CONTIGUOUS(x, m);
/*Quantization entry for mxfp4 experts quantization*/
#define CHECK_TH_CUDA(x, m) \
STD_TORCH_CHECK(x.is_cuda(), m, "must be a CUDA tensor")
#define CHECK_CONTIGUOUS(x, m) \
STD_TORCH_CHECK(x.is_contiguous(), m, "must be contiguous")
#define CHECK_INPUT(x, m) \
CHECK_TH_CUDA(x, m); \
CHECK_CONTIGUOUS(x, m);
constexpr auto HALF = torch::headeronly::ScalarType::Half;
constexpr auto BF16 = torch::headeronly::ScalarType::BFloat16;
@@ -364,12 +373,28 @@ static void validate_mxfp4_experts_quant_inputs(
STD_TORCH_CHECK(output_scale.size(1) * 4 == padded_k);
}
#endif // VLLM_MXFP4_EXPERTS_QUANT_SUPPORTED
static bool mxfp4_experts_quant_sm_supported(int64_t cuda_device_capability) {
#if VLLM_MXFP4_EXPERTS_QUANT_SUPPORTED
return cuda_device_capability >= 100 && cuda_device_capability < 120;
#else
return false;
#endif
}
void mxfp4_experts_quant(
torch::stable::Tensor& output, torch::stable::Tensor& output_scale,
torch::stable::Tensor const& input,
torch::stable::Tensor const& input_offset_by_experts,
torch::stable::Tensor const& output_scale_offset_by_experts,
int64_t n_experts) {
#if VLLM_MXFP4_EXPERTS_QUANT_SUPPORTED
int32_t sm = get_sm_version_num();
STD_TORCH_CHECK(mxfp4_experts_quant_sm_supported(sm),
"No compiled MXFP4 experts quant kernel for SM ", sm,
". Recompile with SM10x/11x FP4 support and CUDA >= 12.9.");
auto m_topk = input.size(0);
auto k = input.size(1);
@@ -390,6 +415,10 @@ void mxfp4_experts_quant(
output_scale_offset_by_experts.data_ptr(), m_topk, k, n_experts,
stream);
});
#else
STD_TORCH_CHECK_NOT_IMPLEMENTED(false,
"MXFP4 experts quant requires CUDA >= 12.9.");
#endif
}
void silu_and_mul_mxfp4_experts_quant(
@@ -398,6 +427,12 @@ void silu_and_mul_mxfp4_experts_quant(
torch::stable::Tensor const& input_offset_by_experts,
torch::stable::Tensor const& output_scale_offset_by_experts,
int64_t n_experts) {
#if VLLM_MXFP4_EXPERTS_QUANT_SUPPORTED
int32_t sm = get_sm_version_num();
STD_TORCH_CHECK(mxfp4_experts_quant_sm_supported(sm),
"No compiled SiLU+Mul MXFP4 experts quant kernel for SM ", sm,
". Recompile with SM10x/11x FP4 support and CUDA >= 12.9.");
auto m_topk = input.size(0);
auto k_times_2 = input.size(1);
STD_TORCH_CHECK(k_times_2 % 2 == 0, "input width must be even (gate || up)");
@@ -420,13 +455,29 @@ void silu_and_mul_mxfp4_experts_quant(
output_scale_offset_by_experts.data_ptr(), m_topk, k, n_experts,
stream);
});
#else
STD_TORCH_CHECK_NOT_IMPLEMENTED(
false, "SiLU+Mul MXFP4 experts quant requires CUDA >= 12.9.");
#endif
}
// Registered here (not torch_bindings.cpp) because VLLM_GPU_FLAGS is applied
// only under COMPILE_LANGUAGE:CUDA, so ENABLE_NVFP4_SM100 is invisible to
// .cpp files and cannot gate the registration from there.
bool mxfp4_experts_quant_supported(int64_t cuda_device_capability) {
return mxfp4_experts_quant_sm_supported(cuda_device_capability);
}
STABLE_TORCH_LIBRARY_FRAGMENT(_C, m) {
m.def("mxfp4_experts_quant_supported(int cuda_device_capability) -> bool");
}
// Registered here so the CUDA 12.8 stub and CUDA 12.9+ implementation stay
// tied to the same translation unit.
STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, m) {
m.impl("mxfp4_experts_quant", TORCH_BOX(&mxfp4_experts_quant));
m.impl("silu_and_mul_mxfp4_experts_quant",
TORCH_BOX(&silu_and_mul_mxfp4_experts_quant));
}
STABLE_TORCH_LIBRARY_IMPL(_C, CompositeExplicitAutograd, m) {
m.impl("mxfp4_experts_quant_supported",
TORCH_BOX(&mxfp4_experts_quant_supported));
}
@@ -22,15 +22,15 @@
#include "../../cuda_vec_utils.cuh"
#if defined(NVFP4_ENABLE_ELTS16) && defined(CUDA_VERSION) && \
CUDA_VERSION >= 12090
#if defined(NVFP4_ENABLE_ELTS16) && defined(CUDART_VERSION) && \
CUDART_VERSION >= 12090
#define ELTS_PER_THREAD 16
#define CVT_FP4_PACK16 1
constexpr int CVT_FP4_ELTS_PER_THREAD = 16;
constexpr bool CVT_FP4_PACK16 = true;
#else
#define ELTS_PER_THREAD 8
#define CVT_FP4_PACK16 0
constexpr int CVT_FP4_ELTS_PER_THREAD = 8;
constexpr bool CVT_FP4_PACK16 = false;
#endif
constexpr int CVT_FP4_SF_VEC_SIZE = 16;
@@ -1,3 +1,4 @@
#include <cuda.h>
#include <cudaTypedefs.h>
#include <torch/csrc/stable/tensor.h>
@@ -174,15 +175,20 @@ bool cutlass_scaled_mm_supports_block_fp8(int64_t cuda_device_capability) {
bool cutlass_group_gemm_supported(int64_t cuda_device_capability) {
// CUTLASS grouped FP8 kernels need at least CUDA 12.3 and SM90 (Hopper)
// or CUDA 12.8 and SM100 (Blackwell)
// or CUDA 12.8 and SM100 (Blackwell). Only report archs that have an
// actual cutlass_moe_mm dispatch compiled into this file.
#if defined CUDA_VERSION
if (cuda_device_capability >= 100) {
#if defined ENABLE_CUTLASS_MOE_SM100 && ENABLE_CUTLASS_MOE_SM100
if (cuda_device_capability >= 100 && cuda_device_capability < 110) {
return CUDA_VERSION >= 12080;
}
if (cuda_device_capability >= 90) {
#endif
#if defined ENABLE_CUTLASS_MOE_SM90 && ENABLE_CUTLASS_MOE_SM90
if (cuda_device_capability >= 90 && cuda_device_capability < 100) {
return CUDA_VERSION >= 12030;
}
#endif
#endif
return false;