[Perf][Qwen4Exp] Keep the HC up projection on the skinny GEMM path (#60027)

Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Stefano Castagnetta
2026-10-05 20:09:36 +08:00
committed by GitHub
co-authored by Thien Tran
parent 710ac56e69
commit 55b80221ed
2 changed files with 25 additions and 5 deletions
@@ -26,6 +26,20 @@ class SkinnyGemmConfig:
static_k: int | None = None
def row_stride_ok(a: torch.Tensor, config: SkinnyGemmConfig) -> bool:
"""Whether the vectorized A loads stay aligned for a row-major ``a``.
The kernel reads A through its layout, so column slices of a wider buffer
are supported without a copy.
"""
vector_bytes = config.vector_width * a.element_size()
return (
a.stride(1) == 1
and a.stride(0) % config.vector_width == 0
and a.data_ptr() % vector_bytes == 0
)
class ShapeDynamicSkinnyGemm:
def __init__(self) -> None:
self._compiled: dict[tuple[torch.dtype, SkinnyGemmConfig, bool], Any] = {}
@@ -209,8 +223,8 @@ class ShapeDynamicSkinnyGemm:
raise ValueError("a and b must have the same BF16 or FP16 dtype")
if not a.is_cuda or not b.is_cuda or a.device != b.device:
raise ValueError("a and b must be CUDA tensors on the same device")
if not a.is_contiguous() or not b.is_contiguous():
raise ValueError("a and b must be contiguous")
if not b.is_contiguous():
raise ValueError("b must be contiguous")
if a.shape[1] != b.shape[1]:
raise ValueError("a and b must have matching K dimensions")
if not 1 <= a.shape[0] <= 16:
@@ -236,6 +250,8 @@ class ShapeDynamicSkinnyGemm:
)
if config.static_k is not None and a.shape[1] != config.static_k:
raise ValueError("input K must match config static_k")
if not row_stride_ok(a, config):
raise ValueError("a must be row-major with vector_width-aligned rows")
has_residual = residual is not None
cache_key = (a.dtype, config, has_residual)
if cache_key not in self._compiled:
@@ -13,6 +13,7 @@ from torch import nn
import vllm.envs as envs
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
SkinnyGemmConfig,
row_stride_ok,
shape_dynamic_skinny_gemm,
)
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
@@ -304,10 +305,13 @@ def _is_packed_row_major(tensor: torch.Tensor) -> bool:
return tensor.dim() == 2 and tensor.stride() == (tensor.shape[1], 1)
def _runtime_ok(x: torch.Tensor, weight: torch.Tensor) -> bool:
def _runtime_ok(
x: torch.Tensor, weight: torch.Tensor, config: SkinnyGemmConfig
) -> bool:
return (
not envs.VLLM_BATCH_INVARIANT
and _is_packed_row_major(x)
and x.dim() == 2
and row_stride_ok(x, config)
and _is_packed_row_major(weight)
and x.dtype == torch.bfloat16
and weight.dtype == torch.bfloat16
@@ -348,7 +352,7 @@ def _qwen4_exp_low_latency_gemm(x: torch.Tensor, weight: torch.Tensor) -> torch.
config = None if plan is None else plan.get(x.shape[0])
if (
config is not None
and _runtime_ok(x, weight)
and _runtime_ok(x, weight, config)
and shape_dynamic_skinny_gemm.is_available()
):
return shape_dynamic_skinny_gemm(x, weight, config)