From 55b80221ed571adc1d6bfcc3edf4b48fa23e9884 Mon Sep 17 00:00:00 2001 From: Stefano Castagnetta Date: Mon, 5 Oct 2026 14:09:36 +0200 Subject: [PATCH] [Perf][Qwen4Exp] Keep the HC up projection on the skinny GEMM path (#60027) Signed-off-by: Stefano Castagnetta Co-authored-by: Thien Tran --- .../kernels/linear/cute_dsl/skinny_gemm.py | 20 +++++++++++++++++-- .../qwen4_exp/nvidia/low_latency_gemm.py | 10 +++++++--- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/vllm/model_executor/kernels/linear/cute_dsl/skinny_gemm.py b/vllm/model_executor/kernels/linear/cute_dsl/skinny_gemm.py index ec1e607caf..e813fe2c5c 100644 --- a/vllm/model_executor/kernels/linear/cute_dsl/skinny_gemm.py +++ b/vllm/model_executor/kernels/linear/cute_dsl/skinny_gemm.py @@ -26,6 +26,20 @@ class SkinnyGemmConfig: static_k: int | None = None +def row_stride_ok(a: torch.Tensor, config: SkinnyGemmConfig) -> bool: + """Whether the vectorized A loads stay aligned for a row-major ``a``. + + The kernel reads A through its layout, so column slices of a wider buffer + are supported without a copy. + """ + vector_bytes = config.vector_width * a.element_size() + return ( + a.stride(1) == 1 + and a.stride(0) % config.vector_width == 0 + and a.data_ptr() % vector_bytes == 0 + ) + + class ShapeDynamicSkinnyGemm: def __init__(self) -> None: self._compiled: dict[tuple[torch.dtype, SkinnyGemmConfig, bool], Any] = {} @@ -209,8 +223,8 @@ class ShapeDynamicSkinnyGemm: raise ValueError("a and b must have the same BF16 or FP16 dtype") if not a.is_cuda or not b.is_cuda or a.device != b.device: raise ValueError("a and b must be CUDA tensors on the same device") - if not a.is_contiguous() or not b.is_contiguous(): - raise ValueError("a and b must be contiguous") + if not b.is_contiguous(): + raise ValueError("b must be contiguous") if a.shape[1] != b.shape[1]: raise ValueError("a and b must have matching K dimensions") if not 1 <= a.shape[0] <= 16: @@ -236,6 +250,8 @@ class ShapeDynamicSkinnyGemm: ) if config.static_k is not None and a.shape[1] != config.static_k: raise ValueError("input K must match config static_k") + if not row_stride_ok(a, config): + raise ValueError("a must be row-major with vector_width-aligned rows") has_residual = residual is not None cache_key = (a.dtype, config, has_residual) if cache_key not in self._compiled: diff --git a/vllm/models/qwen4_exp/nvidia/low_latency_gemm.py b/vllm/models/qwen4_exp/nvidia/low_latency_gemm.py index fd525cdc23..acc5b3e6bd 100644 --- a/vllm/models/qwen4_exp/nvidia/low_latency_gemm.py +++ b/vllm/models/qwen4_exp/nvidia/low_latency_gemm.py @@ -13,6 +13,7 @@ from torch import nn import vllm.envs as envs from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import ( SkinnyGemmConfig, + row_stride_ok, shape_dynamic_skinny_gemm, ) from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod @@ -304,10 +305,13 @@ def _is_packed_row_major(tensor: torch.Tensor) -> bool: return tensor.dim() == 2 and tensor.stride() == (tensor.shape[1], 1) -def _runtime_ok(x: torch.Tensor, weight: torch.Tensor) -> bool: +def _runtime_ok( + x: torch.Tensor, weight: torch.Tensor, config: SkinnyGemmConfig +) -> bool: return ( not envs.VLLM_BATCH_INVARIANT - and _is_packed_row_major(x) + and x.dim() == 2 + and row_stride_ok(x, config) and _is_packed_row_major(weight) and x.dtype == torch.bfloat16 and weight.dtype == torch.bfloat16 @@ -348,7 +352,7 @@ def _qwen4_exp_low_latency_gemm(x: torch.Tensor, weight: torch.Tensor) -> torch. config = None if plan is None else plan.get(x.shape[0]) if ( config is not None - and _runtime_ok(x, weight) + and _runtime_ok(x, weight, config) and shape_dynamic_skinny_gemm.is_available() ): return shape_dynamic_skinny_gemm(x, weight, config)