[Misc] Use functions instead of PTX for the PDL instruction (#46984)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
7fe7fa9cda
commit
d0a2584773
@@ -328,7 +328,7 @@ struct GmemLoaderB {
|
||||
|
||||
__device__ void issue_mainloop() {
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
|
||||
asm volatile("griddepcontrol.wait;");
|
||||
cudaGridDependencySynchronize();
|
||||
#pragma unroll 1
|
||||
for (int loop_idx = 0; loop_idx < k_iter_cnt; loop_idx++) {
|
||||
if (need_wait) {
|
||||
@@ -643,7 +643,7 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel(
|
||||
mma_computer.issue_mainloop();
|
||||
mma_computer.epi();
|
||||
}
|
||||
asm volatile("griddepcontrol.launch_dependents;");
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@@ -100,7 +100,7 @@ __global__ __launch_bounds__(128, 1) void fp32_router_gemm_kernel(
|
||||
}
|
||||
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
asm volatile("griddepcontrol.wait;");
|
||||
cudaGridDependencySynchronize();
|
||||
#endif
|
||||
|
||||
for (int ki = 0; ki < k_iterations; ki++) {
|
||||
@@ -146,7 +146,7 @@ __global__ __launch_bounds__(128, 1) void fp32_router_gemm_kernel(
|
||||
}
|
||||
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
asm volatile("griddepcontrol.launch_dependents;");
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@@ -249,7 +249,7 @@ __global__ void __launch_bounds__(1024)
|
||||
LamportComm<NRanks> comm(params.workspace, params.rank);
|
||||
int clear_access = comm.clear_size / kElemsPerAccess<DType>;
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
asm volatile("griddepcontrol.wait;");
|
||||
cudaGridDependencySynchronize();
|
||||
#endif
|
||||
for (int idx = access_id; idx < tot_access;
|
||||
idx += access_stride, token_id += token_stride) {
|
||||
@@ -313,7 +313,7 @@ __global__ void __launch_bounds__(1024)
|
||||
}
|
||||
comm.update(params.size_q * NRanks);
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
asm volatile("griddepcontrol.launch_dependents;");
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -384,7 +384,7 @@ __global__ void __launch_bounds__(1024)
|
||||
|
||||
DType norm_weight[kElemsPerAccess<DType>]{};
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
asm volatile("griddepcontrol.wait;");
|
||||
cudaGridDependencySynchronize();
|
||||
#endif
|
||||
if (is_q) {
|
||||
if (is_valid_q) {
|
||||
@@ -596,7 +596,7 @@ __global__ void __launch_bounds__(1024)
|
||||
}
|
||||
} // end group loop
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
asm volatile("griddepcontrol.launch_dependents;");
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
|
||||
int clear_access = static_cast<int>(comm.clear_size / kElemsPerAccess<DType>);
|
||||
|
||||
@@ -78,7 +78,7 @@ __global__ __launch_bounds__(128, 1) void router_gemm_kernel_bf16_output(
|
||||
}
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
asm volatile("griddepcontrol.wait;");
|
||||
cudaGridDependencySynchronize();
|
||||
#endif
|
||||
|
||||
// Process the GEMM in chunks
|
||||
@@ -163,7 +163,7 @@ __global__ __launch_bounds__(128, 1) void router_gemm_kernel_bf16_output(
|
||||
}
|
||||
}
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
asm volatile("griddepcontrol.launch_dependents;");
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@@ -78,7 +78,7 @@ __global__ __launch_bounds__(128, 1) void router_gemm_kernel_float_output(
|
||||
}
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
asm volatile("griddepcontrol.wait;");
|
||||
cudaGridDependencySynchronize();
|
||||
#endif
|
||||
|
||||
// Process the GEMM in chunks
|
||||
@@ -163,7 +163,7 @@ __global__ __launch_bounds__(128, 1) void router_gemm_kernel_float_output(
|
||||
}
|
||||
}
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
asm volatile("griddepcontrol.launch_dependents;");
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@@ -48,7 +48,8 @@ static constexpr int NumTopGroupScores = 2;
|
||||
static constexpr int DefaultMaxNumTopExperts = 8;
|
||||
static constexpr int MaxSupportedTopExperts = 22;
|
||||
static constexpr int MaxNumTopGroups = 4;
|
||||
|
||||
// The empirical value for small batch
|
||||
static constexpr int PDLEnableTokens = 16;
|
||||
namespace warp_topk {
|
||||
|
||||
template <int size, typename T>
|
||||
@@ -564,8 +565,8 @@ __global__ void grouped_topk_fused_kernel(
|
||||
T* s_group_scores = reinterpret_cast<T*>(ptr_u);
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
asm volatile("griddepcontrol.wait;"); // I think all prolog can be put before
|
||||
// acqbulk because it's ptr arithmetic
|
||||
cudaGridDependencySynchronize(); // I think all prolog can be put before
|
||||
// acqbulk because it's ptr arithmetic
|
||||
#endif
|
||||
|
||||
// phase 1: per-group scan
|
||||
@@ -609,7 +610,7 @@ __global__ void grouped_topk_fused_kernel(
|
||||
topk_values[i] = 1.0f / static_cast<float>(topk_i32);
|
||||
}
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
asm volatile("griddepcontrol.launch_dependents;");
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
return;
|
||||
}
|
||||
@@ -670,7 +671,7 @@ __global__ void grouped_topk_fused_kernel(
|
||||
}
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
asm volatile("griddepcontrol.launch_dependents;");
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -895,7 +896,8 @@ void invokeNoAuxTc(T* scores, float* topk_values, IdxT* topk_indices,
|
||||
int64_t const num_experts, int64_t const n_group,
|
||||
int64_t const topk_group, int64_t const topk,
|
||||
bool const renormalize, double const routed_scaling_factor,
|
||||
bool enable_pdl = false, cudaStream_t const stream = 0) {
|
||||
const bool enable_pdl = false,
|
||||
cudaStream_t const stream = 0) {
|
||||
cudaLaunchConfig_t config;
|
||||
config.stream = stream;
|
||||
cudaLaunchAttribute attrs[1];
|
||||
@@ -983,7 +985,7 @@ void invokeNoAuxTc(T* scores, float* topk_values, IdxT* topk_indices,
|
||||
int64_t const num_tokens, int64_t const num_experts, \
|
||||
int64_t const n_group, int64_t const topk_group, int64_t const topk, \
|
||||
bool const renormalize, double const routed_scaling_factor, \
|
||||
bool enable_pdl, cudaStream_t const stream);
|
||||
const bool enable_pdl, cudaStream_t const stream);
|
||||
|
||||
INSTANTIATE_NOAUX_TC(float, float, int32_t, SCORING_SIGMOID);
|
||||
INSTANTIATE_NOAUX_TC(float, half, int32_t, SCORING_SIGMOID);
|
||||
@@ -1037,7 +1039,7 @@ std::tuple<torch::stable::Tensor, torch::stable::Tensor> grouped_topk(
|
||||
scores, {num_tokens, topk}, torch::headeronly::ScalarType::Float);
|
||||
auto topk_indices = torch::stable::new_empty(
|
||||
scores, {num_tokens, topk}, torch::headeronly::ScalarType::Int);
|
||||
|
||||
const bool pdl_flag = num_tokens <= vllm::moe::PDLEnableTokens;
|
||||
const cudaStream_t stream =
|
||||
get_current_cuda_stream(scores.get_device_index());
|
||||
auto const sf = static_cast<vllm::moe::ScoringFunc>(scoring_func);
|
||||
@@ -1052,7 +1054,7 @@ std::tuple<torch::stable::Tensor, torch::stable::Tensor> grouped_topk(
|
||||
reinterpret_cast<IdxT*>(topk_indices.mutable_data_ptr()), \
|
||||
reinterpret_cast<BiasT const*>(bias.data_ptr()), num_tokens, \
|
||||
num_experts, n_group, topk_group, topk, renormalize, \
|
||||
routed_scaling_factor, false, stream); \
|
||||
routed_scaling_factor, pdl_flag, stream); \
|
||||
break; \
|
||||
case vllm::moe::SCORING_SIGMOID: \
|
||||
vllm::moe::invokeNoAuxTc<T, BiasT, IdxT, vllm::moe::SCORING_SIGMOID>( \
|
||||
@@ -1061,7 +1063,7 @@ std::tuple<torch::stable::Tensor, torch::stable::Tensor> grouped_topk(
|
||||
reinterpret_cast<IdxT*>(topk_indices.mutable_data_ptr()), \
|
||||
reinterpret_cast<BiasT const*>(bias.data_ptr()), num_tokens, \
|
||||
num_experts, n_group, topk_group, topk, renormalize, \
|
||||
routed_scaling_factor, false, stream); \
|
||||
routed_scaling_factor, pdl_flag, stream); \
|
||||
break; \
|
||||
default: \
|
||||
STD_TORCH_CHECK(false, "Unsupported scoring_func"); \
|
||||
|
||||
@@ -173,7 +173,7 @@ __launch_bounds__(WARPS_PER_CTA* WARP_SIZE_PARAM) __global__
|
||||
float row_chunk[VPT];
|
||||
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
asm volatile("griddepcontrol.wait;");
|
||||
cudaGridDependencySynchronize();
|
||||
#endif
|
||||
|
||||
// NOTE(zhuhaoran): dispatch different input types loading, BF16/FP16 convert
|
||||
@@ -300,7 +300,7 @@ __launch_bounds__(WARPS_PER_CTA* WARP_SIZE_PARAM) __global__
|
||||
}
|
||||
}
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
asm volatile("griddepcontrol.launch_dependents;");
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
return;
|
||||
} else {
|
||||
@@ -425,7 +425,7 @@ __launch_bounds__(WARPS_PER_CTA* WARP_SIZE_PARAM) __global__
|
||||
}
|
||||
}
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900))
|
||||
asm volatile("griddepcontrol.launch_dependents;");
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user