New stable abi cleanup (#46656)

Signed-off-by: Chris Leonard <chleonar@redhat.com>
This commit is contained in:
Chris Leonard
2026-07-03 14:02:26 +08:00
committed by GitHub
parent 2dfaae752b
commit fbc9ba6d30
50 changed files with 96 additions and 44 deletions
@@ -100,6 +100,8 @@ void run_get_group_gemm_starts(
int64_t k = a_tensors.size(1);
int64_t scale_k = cutlass::ceil_div(k, b_group_size);
const torch::stable::accelerator::DeviceGuard device_guard(
a_tensors.get_device_index());
auto stream = get_current_cuda_stream(a_tensors.get_device_index());
if (false) {
@@ -17,11 +17,11 @@
#include <torch/csrc/stable/library.h>
#include <torch/csrc/stable/tensor.h>
#include "libtorch_stable/torch_utils.h"
#include "cutlass_extensions/torch_utils.hpp"
#include "libtorch_stable/cutlass_extensions/torch_utils.hpp"
#include "libtorch_stable/cutlass_extensions/common.hpp"
#include "get_group_starts.cuh"
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
#include "w4a8_utils.cuh"
namespace vllm::cutlass_w4a8_moe {
@@ -6,7 +6,7 @@
#include <torch/csrc/stable/library.h>
#include <torch/csrc/stable/tensor.h>
#include "libtorch_stable/torch_utils.h"
#include "cutlass_extensions/torch_utils.hpp"
#include "libtorch_stable/cutlass_extensions/torch_utils.hpp"
#include "w4a8_utils.cuh"
#include "cutlass/cutlass.h"
@@ -22,7 +22,7 @@
#include "cutlass/util/mixed_dtype_utils.hpp"
#include "libtorch_stable/cutlass_extensions/common.hpp"
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
#include <cuda_runtime.h>
@@ -142,6 +142,8 @@ void mxfp4_run_get_group_gemm_starts(
torch::stable::Tensor const& sf_offsets,
torch::stable::Tensor const& problem_sizes, int M, int N, int K) {
int num_experts = (int)expert_offsets.size(0);
const torch::stable::accelerator::DeviceGuard device_guard(
a_tensors.get_device_index());
auto stream = get_current_cuda_stream(a_tensors.get_device_index());
STD_TORCH_CHECK(out_tensors.size(1) == N,
@@ -172,6 +174,8 @@ void run_mxfp4_blockwise_scaled_group_mm_sm100(
const torch::stable::Tensor& problem_sizes,
const torch::stable::Tensor& expert_offsets,
const torch::stable::Tensor& sf_offsets, int M, int N, int K) {
const torch::stable::accelerator::DeviceGuard device_guard(
a.get_device_index());
using ProblemShape =
cutlass::gemm::GroupProblemShape<Shape<int32_t, int32_t, int32_t>>;
using ElementType = cutlass::float_e2m1_t;
@@ -173,6 +173,8 @@ void run_get_group_gemm_starts(const torch::stable::Tensor& a_starts,
torch::stable::Tensor const& problem_sizes,
int M, int N, int K) {
int num_experts = (int)expert_offsets.size(0);
const torch::stable::accelerator::DeviceGuard device_guard(
a_tensors.get_device_index());
auto stream = get_current_cuda_stream(a_tensors.get_device_index());
STD_TORCH_CHECK(out_tensors.size(1) == N,
@@ -206,6 +208,8 @@ void run_fp4_blockwise_scaled_group_mm_sm100(
const torch::stable::Tensor& problem_sizes,
const torch::stable::Tensor& expert_offsets,
const torch::stable::Tensor& sf_offsets, int M, int N, int K) {
const torch::stable::accelerator::DeviceGuard device_guard(
a.get_device_index());
using ProblemShape =
cutlass::gemm::GroupProblemShape<Shape<int32_t, int32_t, int32_t>>;
using ElementType = cutlass::float_e2m1_t;
@@ -411,6 +415,8 @@ void run_fp4_blockwise_scaled_group_mm_sm120(
const torch::stable::Tensor& problem_sizes,
const torch::stable::Tensor& expert_offsets,
const torch::stable::Tensor& sf_offsets, int M, int N, int K) {
const torch::stable::accelerator::DeviceGuard device_guard(
a.get_device_index());
using ProblemShape =
cutlass::gemm::GroupProblemShape<Shape<int32_t, int32_t, int32_t>>;
using ElementType = cutlass::float_e2m1_t;
@@ -8,7 +8,7 @@
#include "quantization/utils.cuh"
#include "quant_conversions.cuh"
#include "../../../cub_helpers.h"
#include "../../cub_helpers.h"
#include "../../../cuda_compat.h"
namespace vllm {
@@ -150,6 +150,8 @@ void rearrange_kn_weight_as_n32k16_order(
void* b_zero_reorder =
has_zp ? b_zeros_reorder.value().mutable_data_ptr() : nullptr;
const torch::stable::accelerator::DeviceGuard device_guard(
b_qweight.get_device_index());
cudaStream_t stream = get_current_cuda_stream();
if (b_scales.scalar_type() == torch::headeronly::ScalarType::Half) {
allspark::rearrange_kn_weight_as_n32k16_order_ldg16<__half>(
@@ -1,6 +1,6 @@
#pragma once
#include "cutlass_extensions/vllm_collective_builder.cuh"
#include "libtorch_stable/cutlass_extensions/vllm_collective_builder.cuh"
#include "machete_mainloop.cuh"
namespace cutlass::gemm::collective {
@@ -18,9 +18,9 @@
// clang-format on
#include "cutlass_extensions/cute_utils.cuh"
#include "cutlass_extensions/vllm_numeric_conversion.cuh"
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
#include "cutlass_extensions/torch_utils.hpp"
#include "libtorch_stable/cutlass_extensions/vllm_numeric_conversion.cuh"
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
#include "libtorch_stable/cutlass_extensions/torch_utils.hpp"
#include "machete_collective_builder.cuh"
#include "machete_prepacked_layout.cuh"
#include "machete_interleaving_utils.cuh"
@@ -1,7 +1,7 @@
#pragma once
#include "machete_mm_kernel.cuh"
#include "cutlass_extensions/torch_utils.hpp"
#include "libtorch_stable/cutlass_extensions/torch_utils.hpp"
#include "core/scalar_type.hpp"
#include "libtorch_stable/torch_utils.h"
@@ -2,7 +2,7 @@
#include "machete_mm_kernel.cuh"
#include "cutlass_extensions/cute_utils.cuh"
#include "cutlass_extensions/torch_utils.hpp"
#include "libtorch_stable/cutlass_extensions/torch_utils.hpp"
#include <torch/headeronly/util/Exception.h>
namespace machete {
@@ -1,7 +1,7 @@
#pragma once
#include "machete_prepack_kernel.cuh"
#include "cutlass_extensions/torch_utils.hpp"
#include "libtorch_stable/cutlass_extensions/torch_utils.hpp"
#include "core/scalar_type.hpp"
#include "libtorch_stable/torch_utils.h"
@@ -37,6 +37,7 @@ void cutlass_gemm_caller(
typename GemmKernel::MainloopArguments mainloop_args,
typename GemmKernel::EpilogueArguments epilogue_args,
typename GemmKernel::TileSchedulerArguments scheduler = {}) {
const torch::stable::accelerator::DeviceGuard device_guard(device.index());
cutlass::KernelHardwareInfo hw_info;
typename GemmKernel::Arguments args{cutlass::gemm::GemmUniversalMode::kGemm,
prob_shape,
@@ -1,6 +1,6 @@
#include "scaled_mm_kernels.hpp"
#include "scaled_mm_sm90_int8_dispatch.cuh"
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
namespace vllm {
@@ -1,6 +1,6 @@
#include "scaled_mm_kernels.hpp"
#include "scaled_mm_blockwise_sm100_fp8_dispatch.cuh"
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
namespace vllm {
@@ -1,6 +1,6 @@
#include "scaled_mm_kernels.hpp"
#include "scaled_mm_blockwise_sm120_fp8_dispatch.cuh"
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
namespace vllm {
@@ -1,7 +1,7 @@
#include "scaled_mm_kernels.hpp"
#include "scaled_mm_blockwise_sm90_fp8_dispatch.cuh"
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
namespace vllm {
@@ -4,7 +4,7 @@
#include "scaled_mm.cuh"
#include "cutlass_gemm_caller.cuh"
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
/**
* This file defines Gemm kernel configurations for SM100 (fp8) based on the
@@ -1,7 +1,7 @@
#include "scaled_mm_kernels.hpp"
#include "scaled_mm_sm120_fp8_dispatch.cuh"
#include "core/batch_invariant.hpp"
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
namespace vllm {
@@ -4,7 +4,7 @@
#include "scaled_mm.cuh"
#include "cutlass_gemm_caller.cuh"
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
/**
* This file defines Gemm kernel configurations for SM90 (fp8) based on the Gemm
@@ -1,6 +1,6 @@
#include "scaled_mm_kernels.hpp"
#include "scaled_mm_sm90_int8_dispatch.cuh"
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
namespace vllm {
@@ -74,6 +74,8 @@ void run_get_group_gemm_starts(
bool per_act_token = a_scales.numel() != 1;
bool per_out_ch = b_scales.numel() != num_experts;
const torch::stable::accelerator::DeviceGuard device_guard(
a_tensors.get_device_index());
auto stream = get_current_cuda_stream(a_tensors.get_device_index());
if (false) {
@@ -7,8 +7,9 @@
#include "cutlass/gemm/device/gemm_universal_adapter.h"
#include <torch/csrc/stable/ops.h>
#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
#include "libtorch_stable/cutlass_extensions/common.hpp"
#include "libtorch_stable/torch_utils.h"
#include "get_group_starts.cuh"
using namespace cute;
@@ -103,6 +104,8 @@ void cutlass_group_gemm_caller(torch::stable::Tensor& out_tensors,
int num_experts = static_cast<int>(expert_offsets.size(0));
const torch::stable::accelerator::DeviceGuard device_guard(
a_tensors.get_device_index());
auto stream = get_current_cuda_stream(a_tensors.get_device_index());
auto device = a_tensors.device();
@@ -212,6 +212,8 @@ void get_cutlass_moe_mm_problem_sizes_from_expert_offsets_caller(
"n and k must fit in int32");
int const num_experts = static_cast<int>(num_experts64);
const torch::stable::accelerator::DeviceGuard device_guard(
expert_first_token_offset.get_device_index());
auto stream =
get_current_cuda_stream(expert_first_token_offset.get_device_index());
@@ -241,6 +243,7 @@ void get_cutlass_moe_mm_data_caller(
const std::optional<torch::stable::Tensor>& blockscale_offsets,
const bool is_gated) {
auto device = topk_ids.device();
const torch::stable::accelerator::DeviceGuard device_guard(device.index());
auto stream = get_current_cuda_stream(device.index());
torch::stable::Tensor atomic_buffer = torch::stable::new_zeros(
topk_ids, {num_experts}, torch::headeronly::ScalarType::Int);
@@ -311,6 +314,8 @@ void get_cutlass_batched_moe_mm_data_caller(
const torch::stable::Tensor& expert_num_tokens,
const int64_t num_local_experts, const int64_t padded_m, const int64_t n,
const int64_t k) {
const torch::stable::accelerator::DeviceGuard device_guard(
expert_offsets.get_device_index());
auto stream = get_current_cuda_stream(expert_offsets.get_device_index());
if (num_local_experts * padded_m > SWAP_AB_THRESHOLD) {
@@ -156,6 +156,7 @@ inline void cutlass_gemm_caller(torch::stable::Tensor& out,
torch::stable::empty(workspace_size, torch::headeronly::ScalarType::Byte,
std::nullopt, device);
const torch::stable::accelerator::DeviceGuard device_guard(device.index());
auto stream = get_current_cuda_stream(device.index());
CUTLASS_CHECK(gemm_op.can_implement(args));
@@ -1,6 +1,6 @@
#include "../../../../quantization/w8a8/fp8/common.cuh"
#include "../../../dispatch_utils.h"
#include "../../../../cub_helpers.h"
#include "../../../cub_helpers.h"
#include "../../vectorization_utils.cuh"
#include "../../../torch_utils.h"
#include <torch/csrc/stable/macros.h>
@@ -203,6 +203,8 @@ void per_token_group_quant_8bit(const torch::stable::Tensor& input,
STD_TORCH_CHECK(input.numel() % group_size == 0);
STD_TORCH_CHECK(output_s.dim() == 2);
const torch::stable::accelerator::DeviceGuard device_guard(
input.get_device_index());
cudaStream_t stream = get_current_cuda_stream();
constexpr int THREADS_PER_GROUP = 16;
@@ -506,6 +508,8 @@ void per_token_group_quant_8bit_packed(const torch::stable::Tensor& input,
"]; got [", output_s_packed.stride(0), ", ",
output_s_packed.stride(1), "].");
const torch::stable::accelerator::DeviceGuard device_guard(
input.get_device_index());
cudaStream_t stream = get_current_cuda_stream();
constexpr int THREADS_PER_GROUP = 8;
@@ -5,7 +5,7 @@
#include "../../../dispatch_utils.h"
#include "../../../torch_utils.h"
#include "../../vectorization_utils.cuh"
#include "../../../../cub_helpers.h"
#include "../../../cub_helpers.h"
static inline __device__ int8_t float_to_int8_rn(float x) {
#ifdef USE_ROCM