[Kernel] Batch invariant NVFP4 MoE using cutlass (#40372)

Signed-off-by: Jakub Zakrzewski <jzakrzewski@nvidia.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
Jakub Zakrzewski
2026-08-05 04:31:32 +00:00
committed by GitHub
co-authored by OpenAI Codex Wentao Ye
parent 41a7e7da0c
commit c25093a305
5 changed files with 316 additions and 102 deletions
@@ -274,6 +274,15 @@ void run_fp4_blockwise_scaled_group_mm_sm100(
using Gemm1SM = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
using Gemm = Gemm1SM;
static_assert(
cute::is_same_v<
typename Gemm::GemmKernel::TileScheduler,
cutlass::gemm::kernel::detail::PersistentTileSchedulerSm100Group<
ProblemShape, Gemm::GemmKernel::SchedulerPipelineStageCount>>,
"SM100 NVFP4 grouped GEMM must use PersistentTileSchedulerSm100Group "
"for batch invariance (VLLM_BATCH_INVARIANT=1). "
"If you want to change this, add a dedicated config/code path "
"for batch invariant mode, instead of relaxing this check.");
using StrideA = typename Gemm::GemmKernel::InternalStrideA;
using StrideB = typename Gemm::GemmKernel::InternalStrideB;
using StrideC = typename Gemm::GemmKernel::InternalStrideC;
@@ -473,6 +482,15 @@ void run_fp4_blockwise_scaled_group_mm_sm120(
CollectiveEpilogue>;
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
static_assert(
cute::is_base_of_v<
cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperative,
typename Gemm::GemmKernel::CollectiveMainloop::DispatchPolicy::
Schedule>,
"SM120 NVFP4 grouped GEMM must use a cooperative ptr-array mainloop "
"schedule for batch invariance (VLLM_BATCH_INVARIANT=1). "
"If you want to change this, add a dedicated config/code path "
"for batch invariant mode, instead of relaxing this check.");
using StrideA = typename Gemm::GemmKernel::InternalStrideA;
using StrideB = typename Gemm::GemmKernel::InternalStrideB;
using StrideC = typename Gemm::GemmKernel::InternalStrideC;