diff --git a/csrc/cub_helpers.h b/csrc/libtorch_stable/cub_helpers.h similarity index 100% rename from csrc/cub_helpers.h rename to csrc/libtorch_stable/cub_helpers.h diff --git a/csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp b/csrc/libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp similarity index 100% rename from csrc/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp rename to csrc/libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp diff --git a/csrc/cutlass_extensions/torch_utils.hpp b/csrc/libtorch_stable/cutlass_extensions/torch_utils.hpp similarity index 100% rename from csrc/cutlass_extensions/torch_utils.hpp rename to csrc/libtorch_stable/cutlass_extensions/torch_utils.hpp diff --git a/csrc/cutlass_extensions/vllm_collective_builder.cuh b/csrc/libtorch_stable/cutlass_extensions/vllm_collective_builder.cuh similarity index 100% rename from csrc/cutlass_extensions/vllm_collective_builder.cuh rename to csrc/libtorch_stable/cutlass_extensions/vllm_collective_builder.cuh diff --git a/csrc/cutlass_extensions/vllm_numeric_conversion.cuh b/csrc/libtorch_stable/cutlass_extensions/vllm_numeric_conversion.cuh similarity index 100% rename from csrc/cutlass_extensions/vllm_numeric_conversion.cuh rename to csrc/libtorch_stable/cutlass_extensions/vllm_numeric_conversion.cuh diff --git a/csrc/libtorch_stable/dsv3_fused_a_gemm.cu b/csrc/libtorch_stable/dsv3_fused_a_gemm.cu index cbf486fd54..585004c047 100644 --- a/csrc/libtorch_stable/dsv3_fused_a_gemm.cu +++ b/csrc/libtorch_stable/dsv3_fused_a_gemm.cu @@ -733,6 +733,8 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output, output.scalar_type() == torch::headeronly::ScalarType::BFloat16, "Only BFloat16 output dtype is supported"); + const torch::stable::accelerator::DeviceGuard device_guard( + mat_a.get_device_index()); STD_TORCH_CHECK(getSMVersion() >= 90, "required CUDA ARCH >= SM_90"); auto stream = get_current_cuda_stream(mat_a.get_device_index()); diff --git a/csrc/libtorch_stable/fp32_router_gemm_entry.cu b/csrc/libtorch_stable/fp32_router_gemm_entry.cu index b4bc0a11d2..fc09193c86 100644 --- a/csrc/libtorch_stable/fp32_router_gemm_entry.cu +++ b/csrc/libtorch_stable/fp32_router_gemm_entry.cu @@ -127,6 +127,8 @@ void fp32_router_gemm( return; } + const torch::stable::accelerator::DeviceGuard device_guard( + mat_a.get_device_index()); STD_TORCH_CHECK(getSMVersion() >= 90, "fp32_router_gemm: requires SM90+"); auto stream = get_current_cuda_stream(mat_a.get_device_index()); diff --git a/csrc/libtorch_stable/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu b/csrc/libtorch_stable/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu index 4261b702cd..5429ab12d8 100644 --- a/csrc/libtorch_stable/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu +++ b/csrc/libtorch_stable/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu @@ -57,7 +57,7 @@ #include "torch_utils.h" #include "../cuda_compat.h" -#include "../type_convert.cuh" +#include "type_convert.cuh" #include "../attention/dtype_fp8.cuh" #include "dispatch_utils.h" diff --git a/csrc/libtorch_stable/fused_qknorm_rope_kernel.cu b/csrc/libtorch_stable/fused_qknorm_rope_kernel.cu index c9b7ee9e4e..a8e6d32a1b 100644 --- a/csrc/libtorch_stable/fused_qknorm_rope_kernel.cu +++ b/csrc/libtorch_stable/fused_qknorm_rope_kernel.cu @@ -22,7 +22,7 @@ #include "async_util.cuh" #include "../cuda_compat.h" -#include "../type_convert.cuh" +#include "type_convert.cuh" #include "dispatch_utils.h" #define CHECK_TYPE(x, st) \ diff --git a/csrc/libtorch_stable/layernorm_kernels.cu b/csrc/libtorch_stable/layernorm_kernels.cu index de0a103fdf..878b44df93 100644 --- a/csrc/libtorch_stable/layernorm_kernels.cu +++ b/csrc/libtorch_stable/layernorm_kernels.cu @@ -2,9 +2,9 @@ #include "torch_utils.h" -#include "../cub_helpers.h" +#include "cub_helpers.h" #include "../core/batch_invariant.hpp" -#include "../type_convert.cuh" +#include "type_convert.cuh" #include "dispatch_utils.h" #include "quantization/vectorization_utils.cuh" diff --git a/csrc/libtorch_stable/layernorm_quant_kernels.cu b/csrc/libtorch_stable/layernorm_quant_kernels.cu index 26ffa76d6e..f3bf8882e7 100644 --- a/csrc/libtorch_stable/layernorm_quant_kernels.cu +++ b/csrc/libtorch_stable/layernorm_quant_kernels.cu @@ -9,10 +9,10 @@ #include "torch_utils.h" -#include "../cub_helpers.h" +#include "cub_helpers.h" #include "../core/batch_invariant.hpp" #include "../quantization/w8a8/fp8/common.cuh" -#include "../type_convert.cuh" +#include "type_convert.cuh" #include "dispatch_utils.h" #include "quantization/vectorization_utils.cuh" diff --git a/csrc/minimax_reduce_rms_kernel.h b/csrc/libtorch_stable/minimax_reduce_rms_kernel.h similarity index 100% rename from csrc/minimax_reduce_rms_kernel.h rename to csrc/libtorch_stable/minimax_reduce_rms_kernel.h diff --git a/csrc/libtorch_stable/moe/dsv3_router_gemm_entry.cu b/csrc/libtorch_stable/moe/dsv3_router_gemm_entry.cu index 4e06cd3b9a..c5ebaf11f1 100644 --- a/csrc/libtorch_stable/moe/dsv3_router_gemm_entry.cu +++ b/csrc/libtorch_stable/moe/dsv3_router_gemm_entry.cu @@ -151,6 +151,8 @@ void dsv3_router_gemm( output.scalar_type() == torch::headeronly::ScalarType::BFloat16, "output must be float32 or bf16"); + const torch::stable::accelerator::DeviceGuard device_guard( + mat_a.get_device_index()); const int sm = getSMVersion(); STD_TORCH_CHECK(sm >= 90, "required CUDA ARCH >= SM_90"); diff --git a/csrc/libtorch_stable/moe/grouped_topk_kernels.cu b/csrc/libtorch_stable/moe/grouped_topk_kernels.cu index 3b5f98e6ee..da9ef44d03 100644 --- a/csrc/libtorch_stable/moe/grouped_topk_kernels.cu +++ b/csrc/libtorch_stable/moe/grouped_topk_kernels.cu @@ -1040,6 +1040,9 @@ std::tuple grouped_topk( auto topk_indices = torch::stable::new_empty( scores, {num_tokens, topk}, torch::headeronly::ScalarType::Int); const bool pdl_flag = num_tokens <= vllm::moe::PDLEnableTokens; + + const torch::stable::accelerator::DeviceGuard device_guard( + scores.get_device_index()); const cudaStream_t stream = get_current_cuda_stream(scores.get_device_index()); auto const sf = static_cast(scoring_func); diff --git a/csrc/libtorch_stable/moe/moe_align_sum_kernels.cu b/csrc/libtorch_stable/moe/moe_align_sum_kernels.cu index 985c47b076..1fa2c0d18e 100644 --- a/csrc/libtorch_stable/moe/moe_align_sum_kernels.cu +++ b/csrc/libtorch_stable/moe/moe_align_sum_kernels.cu @@ -584,6 +584,8 @@ void moe_align_block_size( torch::stable::Tensor sorted_token_ids, torch::stable::Tensor experts_ids, torch::stable::Tensor num_tokens_post_pad, std::optional maybe_expert_map) { + const torch::stable::accelerator::DeviceGuard device_guard( + topk_ids.get_device_index()); const cudaStream_t stream = get_current_cuda_stream(topk_ids.get_device_index()); @@ -685,6 +687,8 @@ void batched_moe_align_block_size(int64_t max_tokens_per_batch, torch::stable::Tensor num_tokens_post_pad) { namespace batched_kernel = vllm::moe::batched_moe_align_block_size; + const torch::stable::accelerator::DeviceGuard device_guard( + batch_num_tokens.get_device_index()); const cudaStream_t stream = get_current_cuda_stream(batch_num_tokens.get_device_index()); int32_t const B = batch_num_tokens.size(0); @@ -802,6 +806,7 @@ void moe_lora_align_block_size( int device_max_shared_mem; int dev = topk_ids.get_device_index(); + const torch::stable::accelerator::DeviceGuard device_guard(dev); cudaDeviceGetAttribute(&device_max_shared_mem, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev); const cudaStream_t stream = get_current_cuda_stream(dev); diff --git a/csrc/libtorch_stable/moe/moe_permute_unpermute_op.cu b/csrc/libtorch_stable/moe/moe_permute_unpermute_op.cu index b688265eaa..52cf68442f 100644 --- a/csrc/libtorch_stable/moe/moe_permute_unpermute_op.cu +++ b/csrc/libtorch_stable/moe/moe_permute_unpermute_op.cu @@ -87,6 +87,8 @@ void moe_permute_impl( inv_permuted_idx.sizes().equals(token_expert_indices.sizes()), "token_expert_indices shape must be same as inv_permuted_idx"); + const torch::stable::accelerator::DeviceGuard device_guard( + input.get_device_index()); auto device = input.device(); auto n_token = input.sizes()[0]; auto n_hidden = input.sizes()[1]; @@ -182,6 +184,8 @@ void moe_unpermute( permuted_hidden_states.scalar_type() == hidden_states.scalar_type(), "permuted_hidden_states dtype must be same as hidden_states"); + const torch::stable::accelerator::DeviceGuard device_guard( + hidden_states.get_device_index()); auto n_token = hidden_states.size(0); auto n_hidden = hidden_states.size(1); auto stream = get_current_cuda_stream(hidden_states.get_device_index()); @@ -238,6 +242,8 @@ void shuffle_rows(const torch::stable::Tensor& input_tensor, STD_TORCH_CHECK(input_tensor.scalar_type() == output_tensor.scalar_type(), "Input and output tensors must have the same data type"); + const torch::stable::accelerator::DeviceGuard device_guard( + output_tensor.get_device_index()); auto stream = get_current_cuda_stream(output_tensor.get_device_index()); const int64_t blocks = output_tensor.size(0); const int64_t threads = 256; diff --git a/csrc/libtorch_stable/moe/topk_softmax_kernels.cu b/csrc/libtorch_stable/moe/topk_softmax_kernels.cu index e8453579ba..9f2e3640e2 100644 --- a/csrc/libtorch_stable/moe/topk_softmax_kernels.cu +++ b/csrc/libtorch_stable/moe/topk_softmax_kernels.cu @@ -25,7 +25,7 @@ #include #include "../../cuda_compat.h" -#include "../../cub_helpers.h" +#include "../cub_helpers.h" #include "libtorch_stable/torch_utils.h" #ifndef USE_ROCM diff --git a/csrc/libtorch_stable/moe/topk_softplus_sqrt_kernels.cu b/csrc/libtorch_stable/moe/topk_softplus_sqrt_kernels.cu index 0f85f9b4d3..095a766783 100644 --- a/csrc/libtorch_stable/moe/topk_softplus_sqrt_kernels.cu +++ b/csrc/libtorch_stable/moe/topk_softplus_sqrt_kernels.cu @@ -26,7 +26,7 @@ #include #include "../../cuda_compat.h" -#include "../../cub_helpers.h" +#include "../cub_helpers.h" #include "libtorch_stable/torch_utils.h" #ifndef USE_ROCM #include diff --git a/csrc/libtorch_stable/quantization/cutlass_w4a8/get_group_starts.cuh b/csrc/libtorch_stable/quantization/cutlass_w4a8/get_group_starts.cuh index 5cda4c9750..7217617912 100644 --- a/csrc/libtorch_stable/quantization/cutlass_w4a8/get_group_starts.cuh +++ b/csrc/libtorch_stable/quantization/cutlass_w4a8/get_group_starts.cuh @@ -100,6 +100,8 @@ void run_get_group_gemm_starts( int64_t k = a_tensors.size(1); int64_t scale_k = cutlass::ceil_div(k, b_group_size); + const torch::stable::accelerator::DeviceGuard device_guard( + a_tensors.get_device_index()); auto stream = get_current_cuda_stream(a_tensors.get_device_index()); if (false) { diff --git a/csrc/libtorch_stable/quantization/cutlass_w4a8/w4a8_grouped_mm_entry.cu b/csrc/libtorch_stable/quantization/cutlass_w4a8/w4a8_grouped_mm_entry.cu index 53ffe52136..1d7eb04093 100644 --- a/csrc/libtorch_stable/quantization/cutlass_w4a8/w4a8_grouped_mm_entry.cu +++ b/csrc/libtorch_stable/quantization/cutlass_w4a8/w4a8_grouped_mm_entry.cu @@ -17,11 +17,11 @@ #include #include #include "libtorch_stable/torch_utils.h" -#include "cutlass_extensions/torch_utils.hpp" +#include "libtorch_stable/cutlass_extensions/torch_utils.hpp" #include "libtorch_stable/cutlass_extensions/common.hpp" #include "get_group_starts.cuh" -#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" +#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" #include "w4a8_utils.cuh" namespace vllm::cutlass_w4a8_moe { diff --git a/csrc/libtorch_stable/quantization/cutlass_w4a8/w4a8_mm_entry.cu b/csrc/libtorch_stable/quantization/cutlass_w4a8/w4a8_mm_entry.cu index 502f430b30..8f19394b39 100644 --- a/csrc/libtorch_stable/quantization/cutlass_w4a8/w4a8_mm_entry.cu +++ b/csrc/libtorch_stable/quantization/cutlass_w4a8/w4a8_mm_entry.cu @@ -6,7 +6,7 @@ #include #include #include "libtorch_stable/torch_utils.h" -#include "cutlass_extensions/torch_utils.hpp" +#include "libtorch_stable/cutlass_extensions/torch_utils.hpp" #include "w4a8_utils.cuh" #include "cutlass/cutlass.h" @@ -22,7 +22,7 @@ #include "cutlass/util/mixed_dtype_utils.hpp" #include "libtorch_stable/cutlass_extensions/common.hpp" -#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" +#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" #include diff --git a/csrc/libtorch_stable/quantization/fp4/mxfp4_blockwise_moe_kernel.cu b/csrc/libtorch_stable/quantization/fp4/mxfp4_blockwise_moe_kernel.cu index 04e98b6076..e4d2f22012 100644 --- a/csrc/libtorch_stable/quantization/fp4/mxfp4_blockwise_moe_kernel.cu +++ b/csrc/libtorch_stable/quantization/fp4/mxfp4_blockwise_moe_kernel.cu @@ -142,6 +142,8 @@ void mxfp4_run_get_group_gemm_starts( torch::stable::Tensor const& sf_offsets, torch::stable::Tensor const& problem_sizes, int M, int N, int K) { int num_experts = (int)expert_offsets.size(0); + const torch::stable::accelerator::DeviceGuard device_guard( + a_tensors.get_device_index()); auto stream = get_current_cuda_stream(a_tensors.get_device_index()); STD_TORCH_CHECK(out_tensors.size(1) == N, @@ -172,6 +174,8 @@ void run_mxfp4_blockwise_scaled_group_mm_sm100( const torch::stable::Tensor& problem_sizes, const torch::stable::Tensor& expert_offsets, const torch::stable::Tensor& sf_offsets, int M, int N, int K) { + const torch::stable::accelerator::DeviceGuard device_guard( + a.get_device_index()); using ProblemShape = cutlass::gemm::GroupProblemShape>; using ElementType = cutlass::float_e2m1_t; diff --git a/csrc/libtorch_stable/quantization/fp4/nvfp4_blockwise_moe_kernel.cu b/csrc/libtorch_stable/quantization/fp4/nvfp4_blockwise_moe_kernel.cu index 88caf03fda..b044db6d32 100644 --- a/csrc/libtorch_stable/quantization/fp4/nvfp4_blockwise_moe_kernel.cu +++ b/csrc/libtorch_stable/quantization/fp4/nvfp4_blockwise_moe_kernel.cu @@ -173,6 +173,8 @@ void run_get_group_gemm_starts(const torch::stable::Tensor& a_starts, torch::stable::Tensor const& problem_sizes, int M, int N, int K) { int num_experts = (int)expert_offsets.size(0); + const torch::stable::accelerator::DeviceGuard device_guard( + a_tensors.get_device_index()); auto stream = get_current_cuda_stream(a_tensors.get_device_index()); STD_TORCH_CHECK(out_tensors.size(1) == N, @@ -206,6 +208,8 @@ void run_fp4_blockwise_scaled_group_mm_sm100( const torch::stable::Tensor& problem_sizes, const torch::stable::Tensor& expert_offsets, const torch::stable::Tensor& sf_offsets, int M, int N, int K) { + const torch::stable::accelerator::DeviceGuard device_guard( + a.get_device_index()); using ProblemShape = cutlass::gemm::GroupProblemShape>; using ElementType = cutlass::float_e2m1_t; @@ -411,6 +415,8 @@ void run_fp4_blockwise_scaled_group_mm_sm120( const torch::stable::Tensor& problem_sizes, const torch::stable::Tensor& expert_offsets, const torch::stable::Tensor& sf_offsets, int M, int N, int K) { + const torch::stable::accelerator::DeviceGuard device_guard( + a.get_device_index()); using ProblemShape = cutlass::gemm::GroupProblemShape>; using ElementType = cutlass::float_e2m1_t; diff --git a/csrc/libtorch_stable/quantization/fused_kernels/layernorm_utils.cuh b/csrc/libtorch_stable/quantization/fused_kernels/layernorm_utils.cuh index 290abedcf9..0139cfbc95 100644 --- a/csrc/libtorch_stable/quantization/fused_kernels/layernorm_utils.cuh +++ b/csrc/libtorch_stable/quantization/fused_kernels/layernorm_utils.cuh @@ -8,7 +8,7 @@ #include "quantization/utils.cuh" #include "quant_conversions.cuh" -#include "../../../cub_helpers.h" +#include "../../cub_helpers.h" #include "../../../cuda_compat.h" namespace vllm { diff --git a/csrc/libtorch_stable/quantization/gptq_allspark/allspark_repack.cu b/csrc/libtorch_stable/quantization/gptq_allspark/allspark_repack.cu index b325d30a04..9a00f66241 100644 --- a/csrc/libtorch_stable/quantization/gptq_allspark/allspark_repack.cu +++ b/csrc/libtorch_stable/quantization/gptq_allspark/allspark_repack.cu @@ -150,6 +150,8 @@ void rearrange_kn_weight_as_n32k16_order( void* b_zero_reorder = has_zp ? b_zeros_reorder.value().mutable_data_ptr() : nullptr; + const torch::stable::accelerator::DeviceGuard device_guard( + b_qweight.get_device_index()); cudaStream_t stream = get_current_cuda_stream(); if (b_scales.scalar_type() == torch::headeronly::ScalarType::Half) { allspark::rearrange_kn_weight_as_n32k16_order_ldg16<__half>( diff --git a/csrc/libtorch_stable/quantization/machete/machete_collective_builder.cuh b/csrc/libtorch_stable/quantization/machete/machete_collective_builder.cuh index ee825583de..ba8da0af2c 100644 --- a/csrc/libtorch_stable/quantization/machete/machete_collective_builder.cuh +++ b/csrc/libtorch_stable/quantization/machete/machete_collective_builder.cuh @@ -1,6 +1,6 @@ #pragma once -#include "cutlass_extensions/vllm_collective_builder.cuh" +#include "libtorch_stable/cutlass_extensions/vllm_collective_builder.cuh" #include "machete_mainloop.cuh" namespace cutlass::gemm::collective { diff --git a/csrc/libtorch_stable/quantization/machete/machete_mm_kernel.cuh b/csrc/libtorch_stable/quantization/machete/machete_mm_kernel.cuh index db3321a39d..57655f369c 100644 --- a/csrc/libtorch_stable/quantization/machete/machete_mm_kernel.cuh +++ b/csrc/libtorch_stable/quantization/machete/machete_mm_kernel.cuh @@ -18,9 +18,9 @@ // clang-format on #include "cutlass_extensions/cute_utils.cuh" -#include "cutlass_extensions/vllm_numeric_conversion.cuh" -#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" -#include "cutlass_extensions/torch_utils.hpp" +#include "libtorch_stable/cutlass_extensions/vllm_numeric_conversion.cuh" +#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" +#include "libtorch_stable/cutlass_extensions/torch_utils.hpp" #include "machete_collective_builder.cuh" #include "machete_prepacked_layout.cuh" #include "machete_interleaving_utils.cuh" diff --git a/csrc/libtorch_stable/quantization/machete/machete_mm_launcher.cuh b/csrc/libtorch_stable/quantization/machete/machete_mm_launcher.cuh index fcf7f18aac..5ecf03daf7 100644 --- a/csrc/libtorch_stable/quantization/machete/machete_mm_launcher.cuh +++ b/csrc/libtorch_stable/quantization/machete/machete_mm_launcher.cuh @@ -1,7 +1,7 @@ #pragma once #include "machete_mm_kernel.cuh" -#include "cutlass_extensions/torch_utils.hpp" +#include "libtorch_stable/cutlass_extensions/torch_utils.hpp" #include "core/scalar_type.hpp" #include "libtorch_stable/torch_utils.h" diff --git a/csrc/libtorch_stable/quantization/machete/machete_prepack_kernel.cuh b/csrc/libtorch_stable/quantization/machete/machete_prepack_kernel.cuh index e1e054e5a0..6a28dceccc 100644 --- a/csrc/libtorch_stable/quantization/machete/machete_prepack_kernel.cuh +++ b/csrc/libtorch_stable/quantization/machete/machete_prepack_kernel.cuh @@ -2,7 +2,7 @@ #include "machete_mm_kernel.cuh" #include "cutlass_extensions/cute_utils.cuh" -#include "cutlass_extensions/torch_utils.hpp" +#include "libtorch_stable/cutlass_extensions/torch_utils.hpp" #include namespace machete { diff --git a/csrc/libtorch_stable/quantization/machete/machete_prepack_launcher.cuh b/csrc/libtorch_stable/quantization/machete/machete_prepack_launcher.cuh index 94f6f684bc..63af5e496b 100644 --- a/csrc/libtorch_stable/quantization/machete/machete_prepack_launcher.cuh +++ b/csrc/libtorch_stable/quantization/machete/machete_prepack_launcher.cuh @@ -1,7 +1,7 @@ #pragma once #include "machete_prepack_kernel.cuh" -#include "cutlass_extensions/torch_utils.hpp" +#include "libtorch_stable/cutlass_extensions/torch_utils.hpp" #include "core/scalar_type.hpp" #include "libtorch_stable/torch_utils.h" diff --git a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/cutlass_gemm_caller.cuh b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/cutlass_gemm_caller.cuh index 1d9023484f..a93a136bf4 100644 --- a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/cutlass_gemm_caller.cuh +++ b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/cutlass_gemm_caller.cuh @@ -37,6 +37,7 @@ void cutlass_gemm_caller( typename GemmKernel::MainloopArguments mainloop_args, typename GemmKernel::EpilogueArguments epilogue_args, typename GemmKernel::TileSchedulerArguments scheduler = {}) { + const torch::stable::accelerator::DeviceGuard device_guard(device.index()); cutlass::KernelHardwareInfo hw_info; typename GemmKernel::Arguments args{cutlass::gemm::GemmUniversalMode::kGemm, prob_shape, diff --git a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_azp_sm90_int8.cu b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_azp_sm90_int8.cu index bc088cf633..5178eed072 100644 --- a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_azp_sm90_int8.cu +++ b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_azp_sm90_int8.cu @@ -1,6 +1,6 @@ #include "scaled_mm_kernels.hpp" #include "scaled_mm_sm90_int8_dispatch.cuh" -#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" +#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" namespace vllm { diff --git a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm100_fp8.cu b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm100_fp8.cu index f3df69850e..b793001226 100644 --- a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm100_fp8.cu +++ b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm100_fp8.cu @@ -1,6 +1,6 @@ #include "scaled_mm_kernels.hpp" #include "scaled_mm_blockwise_sm100_fp8_dispatch.cuh" -#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" +#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" namespace vllm { diff --git a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm120_fp8.cu b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm120_fp8.cu index 7ceb0697df..426025ac13 100644 --- a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm120_fp8.cu +++ b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm120_fp8.cu @@ -1,6 +1,6 @@ #include "scaled_mm_kernels.hpp" #include "scaled_mm_blockwise_sm120_fp8_dispatch.cuh" -#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" +#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" namespace vllm { diff --git a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm90_fp8.cu b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm90_fp8.cu index d3318c4876..a97909d37a 100644 --- a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm90_fp8.cu +++ b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_blockwise_sm90_fp8.cu @@ -1,7 +1,7 @@ #include "scaled_mm_kernels.hpp" #include "scaled_mm_blockwise_sm90_fp8_dispatch.cuh" -#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" +#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" namespace vllm { diff --git a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_sm100_fp8_dispatch.cuh b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_sm100_fp8_dispatch.cuh index f790b3653d..42f9e4d5bc 100644 --- a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_sm100_fp8_dispatch.cuh +++ b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_sm100_fp8_dispatch.cuh @@ -4,7 +4,7 @@ #include "scaled_mm.cuh" #include "cutlass_gemm_caller.cuh" -#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" +#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" /** * This file defines Gemm kernel configurations for SM100 (fp8) based on the diff --git a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_sm120_fp8.cu b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_sm120_fp8.cu index 972d6c6260..1f709699eb 100644 --- a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_sm120_fp8.cu +++ b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_sm120_fp8.cu @@ -1,7 +1,7 @@ #include "scaled_mm_kernels.hpp" #include "scaled_mm_sm120_fp8_dispatch.cuh" #include "core/batch_invariant.hpp" -#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" +#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" namespace vllm { diff --git a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_sm90_fp8_dispatch.cuh b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_sm90_fp8_dispatch.cuh index f78b8daea5..2fae3016c3 100644 --- a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_sm90_fp8_dispatch.cuh +++ b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_sm90_fp8_dispatch.cuh @@ -4,7 +4,7 @@ #include "scaled_mm.cuh" #include "cutlass_gemm_caller.cuh" -#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" +#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" /** * This file defines Gemm kernel configurations for SM90 (fp8) based on the Gemm diff --git a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_sm90_int8.cu b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_sm90_int8.cu index 717a2a5883..bf2bd030e3 100644 --- a/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_sm90_int8.cu +++ b/csrc/libtorch_stable/quantization/w8a8/cutlass/c3x/scaled_mm_sm90_int8.cu @@ -1,6 +1,6 @@ #include "scaled_mm_kernels.hpp" #include "scaled_mm_sm90_int8_dispatch.cuh" -#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" +#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" namespace vllm { diff --git a/csrc/libtorch_stable/quantization/w8a8/cutlass/moe/get_group_starts.cuh b/csrc/libtorch_stable/quantization/w8a8/cutlass/moe/get_group_starts.cuh index e073b4e64a..6942c4b48c 100644 --- a/csrc/libtorch_stable/quantization/w8a8/cutlass/moe/get_group_starts.cuh +++ b/csrc/libtorch_stable/quantization/w8a8/cutlass/moe/get_group_starts.cuh @@ -74,6 +74,8 @@ void run_get_group_gemm_starts( bool per_act_token = a_scales.numel() != 1; bool per_out_ch = b_scales.numel() != num_experts; + const torch::stable::accelerator::DeviceGuard device_guard( + a_tensors.get_device_index()); auto stream = get_current_cuda_stream(a_tensors.get_device_index()); if (false) { diff --git a/csrc/libtorch_stable/quantization/w8a8/cutlass/moe/grouped_mm_c3x.cuh b/csrc/libtorch_stable/quantization/w8a8/cutlass/moe/grouped_mm_c3x.cuh index b523d7baea..b4cd520e96 100644 --- a/csrc/libtorch_stable/quantization/w8a8/cutlass/moe/grouped_mm_c3x.cuh +++ b/csrc/libtorch_stable/quantization/w8a8/cutlass/moe/grouped_mm_c3x.cuh @@ -7,8 +7,9 @@ #include "cutlass/gemm/device/gemm_universal_adapter.h" #include -#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" +#include "libtorch_stable/cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp" #include "libtorch_stable/cutlass_extensions/common.hpp" +#include "libtorch_stable/torch_utils.h" #include "get_group_starts.cuh" using namespace cute; @@ -103,6 +104,8 @@ void cutlass_group_gemm_caller(torch::stable::Tensor& out_tensors, int num_experts = static_cast(expert_offsets.size(0)); + const torch::stable::accelerator::DeviceGuard device_guard( + a_tensors.get_device_index()); auto stream = get_current_cuda_stream(a_tensors.get_device_index()); auto device = a_tensors.device(); diff --git a/csrc/libtorch_stable/quantization/w8a8/cutlass/moe/moe_data.cu b/csrc/libtorch_stable/quantization/w8a8/cutlass/moe/moe_data.cu index 2632989cc6..47e0985b12 100644 --- a/csrc/libtorch_stable/quantization/w8a8/cutlass/moe/moe_data.cu +++ b/csrc/libtorch_stable/quantization/w8a8/cutlass/moe/moe_data.cu @@ -212,6 +212,8 @@ void get_cutlass_moe_mm_problem_sizes_from_expert_offsets_caller( "n and k must fit in int32"); int const num_experts = static_cast(num_experts64); + const torch::stable::accelerator::DeviceGuard device_guard( + expert_first_token_offset.get_device_index()); auto stream = get_current_cuda_stream(expert_first_token_offset.get_device_index()); @@ -241,6 +243,7 @@ void get_cutlass_moe_mm_data_caller( const std::optional& blockscale_offsets, const bool is_gated) { auto device = topk_ids.device(); + const torch::stable::accelerator::DeviceGuard device_guard(device.index()); auto stream = get_current_cuda_stream(device.index()); torch::stable::Tensor atomic_buffer = torch::stable::new_zeros( topk_ids, {num_experts}, torch::headeronly::ScalarType::Int); @@ -311,6 +314,8 @@ void get_cutlass_batched_moe_mm_data_caller( const torch::stable::Tensor& expert_num_tokens, const int64_t num_local_experts, const int64_t padded_m, const int64_t n, const int64_t k) { + const torch::stable::accelerator::DeviceGuard device_guard( + expert_offsets.get_device_index()); auto stream = get_current_cuda_stream(expert_offsets.get_device_index()); if (num_local_experts * padded_m > SWAP_AB_THRESHOLD) { diff --git a/csrc/libtorch_stable/quantization/w8a8/cutlass/scaled_mm_c2x.cuh b/csrc/libtorch_stable/quantization/w8a8/cutlass/scaled_mm_c2x.cuh index d2b54cb911..9f9e12e982 100644 --- a/csrc/libtorch_stable/quantization/w8a8/cutlass/scaled_mm_c2x.cuh +++ b/csrc/libtorch_stable/quantization/w8a8/cutlass/scaled_mm_c2x.cuh @@ -156,6 +156,7 @@ inline void cutlass_gemm_caller(torch::stable::Tensor& out, torch::stable::empty(workspace_size, torch::headeronly::ScalarType::Byte, std::nullopt, device); + const torch::stable::accelerator::DeviceGuard device_guard(device.index()); auto stream = get_current_cuda_stream(device.index()); CUTLASS_CHECK(gemm_op.can_implement(args)); diff --git a/csrc/libtorch_stable/quantization/w8a8/fp8/common.cu b/csrc/libtorch_stable/quantization/w8a8/fp8/common.cu index d02fc2296e..86696e2f7a 100644 --- a/csrc/libtorch_stable/quantization/w8a8/fp8/common.cu +++ b/csrc/libtorch_stable/quantization/w8a8/fp8/common.cu @@ -1,6 +1,6 @@ #include "../../../../quantization/w8a8/fp8/common.cuh" #include "../../../dispatch_utils.h" -#include "../../../../cub_helpers.h" +#include "../../../cub_helpers.h" #include "../../vectorization_utils.cuh" #include "../../../torch_utils.h" #include diff --git a/csrc/libtorch_stable/quantization/w8a8/fp8/per_token_group_quant.cu b/csrc/libtorch_stable/quantization/w8a8/fp8/per_token_group_quant.cu index 15d806cb4c..f4a0bd5442 100644 --- a/csrc/libtorch_stable/quantization/w8a8/fp8/per_token_group_quant.cu +++ b/csrc/libtorch_stable/quantization/w8a8/fp8/per_token_group_quant.cu @@ -203,6 +203,8 @@ void per_token_group_quant_8bit(const torch::stable::Tensor& input, STD_TORCH_CHECK(input.numel() % group_size == 0); STD_TORCH_CHECK(output_s.dim() == 2); + const torch::stable::accelerator::DeviceGuard device_guard( + input.get_device_index()); cudaStream_t stream = get_current_cuda_stream(); constexpr int THREADS_PER_GROUP = 16; @@ -506,6 +508,8 @@ void per_token_group_quant_8bit_packed(const torch::stable::Tensor& input, "]; got [", output_s_packed.stride(0), ", ", output_s_packed.stride(1), "]."); + const torch::stable::accelerator::DeviceGuard device_guard( + input.get_device_index()); cudaStream_t stream = get_current_cuda_stream(); constexpr int THREADS_PER_GROUP = 8; diff --git a/csrc/libtorch_stable/quantization/w8a8/int8/scaled_quant.cu b/csrc/libtorch_stable/quantization/w8a8/int8/scaled_quant.cu index ede7913a35..e620c18184 100644 --- a/csrc/libtorch_stable/quantization/w8a8/int8/scaled_quant.cu +++ b/csrc/libtorch_stable/quantization/w8a8/int8/scaled_quant.cu @@ -5,7 +5,7 @@ #include "../../../dispatch_utils.h" #include "../../../torch_utils.h" #include "../../vectorization_utils.cuh" -#include "../../../../cub_helpers.h" +#include "../../../cub_helpers.h" static inline __device__ int8_t float_to_int8_rn(float x) { #ifdef USE_ROCM diff --git a/csrc/libtorch_stable/sampler.cu b/csrc/libtorch_stable/sampler.cu index 68848b8456..519e213281 100644 --- a/csrc/libtorch_stable/sampler.cu +++ b/csrc/libtorch_stable/sampler.cu @@ -665,6 +665,8 @@ void top_k_per_row_decode(const torch::stable::Tensor& logits, int64_t next_n, constexpr int kSortingAlgorithmThreshold = 12288; constexpr int kSplitWorkThreshold = 200 * 1000; constexpr int kNumThreadsPerBlock = 512; + const torch::stable::accelerator::DeviceGuard device_guard( + logits.get_device_index()); const cudaStream_t stream = get_current_cuda_stream(); const auto numColumns = logits.size(1); @@ -727,6 +729,8 @@ void top_k_per_row_prefill(const torch::stable::Tensor& logits, int64_t stride0, int64_t stride1, int64_t topK) { constexpr int kSortingAlgorithmThreshold = 12288; constexpr int kNumThreadsPerBlock = 512; + const torch::stable::accelerator::DeviceGuard device_guard( + logits.get_device_index()); const cudaStream_t stream = get_current_cuda_stream(); int numInsertionBlocks = diff --git a/csrc/libtorch_stable/topk.cu b/csrc/libtorch_stable/topk.cu index 7656ba8cf8..07b7521f86 100644 --- a/csrc/libtorch_stable/topk.cu +++ b/csrc/libtorch_stable/topk.cu @@ -21,6 +21,8 @@ void launch_persistent_topk(const torch::stable::Tensor& logits, int64_t max_seq_len) { namespace P = vllm::persistent; + const torch::stable::accelerator::DeviceGuard device_guard( + logits.get_device_index()); const int64_t num_rows = logits.size(0); const int64_t stride = logits.stride(0); const cudaStream_t stream = get_current_cuda_stream(); @@ -260,6 +262,9 @@ void persistent_topk(const torch::stable::Tensor& logits, k == 512 || k == 1024 || k == 2048, "persistent_topk supports k=512, k=1024, or k=2048, got k=", k); + const torch::stable::accelerator::DeviceGuard device_guard( + logits.get_device_index()); + if (k == 512) { launch_persistent_topk<512>(logits, lengths, output, workspace, max_seq_len); diff --git a/csrc/type_convert.cuh b/csrc/libtorch_stable/type_convert.cuh similarity index 100% rename from csrc/type_convert.cuh rename to csrc/libtorch_stable/type_convert.cuh diff --git a/vllm/platforms/cuda.py b/vllm/platforms/cuda.py index ee73eef879..605650525a 100644 --- a/vllm/platforms/cuda.py +++ b/vllm/platforms/cuda.py @@ -6,6 +6,7 @@ pynvml. However, it should not initialize cuda context. from __future__ import annotations +import contextlib import os import platform from collections.abc import Callable @@ -20,6 +21,9 @@ from typing_extensions import ParamSpec # import custom ops, trigger op registration import vllm._C_stable_libtorch # noqa + +with contextlib.suppress(ImportError): + import vllm._qutlass_C # noqa import vllm.envs as envs from vllm.logger import init_logger from vllm.utils.import_utils import import_pynvml @@ -39,11 +43,6 @@ else: logger = init_logger(__name__) -try: - import vllm._qutlass_C # noqa: F401 -except ImportError as e: - logger.warning("Failed to import from vllm._qutlass_C: %r", e) - _P = ParamSpec("_P") _R = TypeVar("_R") @@ -221,16 +220,10 @@ class CudaPlatformBase(Platform): import vllm._C_stable_libtorch # noqa: F401 except ImportError as e: logger.warning_once("Failed to import from vllm._C_stable_libtorch: %r", e) - try: + with contextlib.suppress(ImportError): import vllm._moe_C_stable_libtorch # noqa: F401 - except ImportError as e: - logger.warning_once( - "Failed to import from vllm._moe_C_stable_libtorch: %r", e - ) - try: + with contextlib.suppress(ImportError): import vllm._qutlass_C # noqa: F401 - except ImportError as e: - logger.warning_once("Failed to import from vllm._qutlass_C: %r", e) @property def supported_dtypes(self) -> list[torch.dtype]: