diff --git a/csrc/libtorch_stable/kimi_k3/attn_res_kernel.cu b/csrc/libtorch_stable/kimi_k3/attn_res_kernel.cu index eb4dcf6bf5..f92a97bf06 100644 --- a/csrc/libtorch_stable/kimi_k3/attn_res_kernel.cu +++ b/csrc/libtorch_stable/kimi_k3/attn_res_kernel.cu @@ -914,6 +914,13 @@ void kimi_k3_attn_res(torch::stable::Tensor& prefix, cudaDeviceProp const* properties = get_device_prop(); STD_TORCH_CHECK(properties->major == 10, "Kimi K3 AttnRes requires the SM100 family"); + int64_t const hidden_size = prefix.size(1); + STD_TORCH_CHECK(prefix.stride(0) == hidden_size && + delta.stride(0) == hidden_size && + output.stride(0) == hidden_size, + "Kimi K3 AttnRes requires densely packed rows; got strides ", + prefix.stride(0), ", ", delta.stride(0), ", ", + output.stride(0), " for hidden_size ", hidden_size); using namespace sm100::fwd_prod_v2; // Two-source chunks and two resident CTAs are beneficial once setup is diff --git a/vllm/models/kimi_k3/nvidia/ops/attn_res.py b/vllm/models/kimi_k3/nvidia/ops/attn_res.py index 01078d6c01..06f73ce8f7 100644 --- a/vllm/models/kimi_k3/nvidia/ops/attn_res.py +++ b/vllm/models/kimi_k3/nvidia/ops/attn_res.py @@ -187,7 +187,9 @@ def attn_res( assert qk_weight.stride(-1) == 1 assert output_norm_weight is None or output_norm_weight.stride(-1) == 1 # The in-tree NVIDIA kernel covers the common fused-add + output-norm path; - # Triton handles block boundaries and final pre-norm output. + # Triton handles block boundaries and final pre-norm output. The native op + # is only compiled for SM100 under CUDA >= 13, so a device check alone is + # not enough to know it exists. if ( hidden_size == 7168 and delta is not None @@ -195,6 +197,7 @@ def attn_res( and num_blocks > 0 and block_write_idx < 0 and current_platform.is_device_capability_family(100) + and hasattr(torch.ops._C, "kimi_k3_attn_res") ): return ops.kimi_k3_attn_res( prefix,