[Perf][Qwen4Exp] Keep the HC up projection on the skinny GEMM path (#60027)
Signed-off-by: Stefano Castagnetta <scastagnetta@nvidia.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
co-authored by
Thien Tran
parent
710ac56e69
commit
55b80221ed
@@ -26,6 +26,20 @@ class SkinnyGemmConfig:
|
||||
static_k: int | None = None
|
||||
|
||||
|
||||
def row_stride_ok(a: torch.Tensor, config: SkinnyGemmConfig) -> bool:
|
||||
"""Whether the vectorized A loads stay aligned for a row-major ``a``.
|
||||
|
||||
The kernel reads A through its layout, so column slices of a wider buffer
|
||||
are supported without a copy.
|
||||
"""
|
||||
vector_bytes = config.vector_width * a.element_size()
|
||||
return (
|
||||
a.stride(1) == 1
|
||||
and a.stride(0) % config.vector_width == 0
|
||||
and a.data_ptr() % vector_bytes == 0
|
||||
)
|
||||
|
||||
|
||||
class ShapeDynamicSkinnyGemm:
|
||||
def __init__(self) -> None:
|
||||
self._compiled: dict[tuple[torch.dtype, SkinnyGemmConfig, bool], Any] = {}
|
||||
@@ -209,8 +223,8 @@ class ShapeDynamicSkinnyGemm:
|
||||
raise ValueError("a and b must have the same BF16 or FP16 dtype")
|
||||
if not a.is_cuda or not b.is_cuda or a.device != b.device:
|
||||
raise ValueError("a and b must be CUDA tensors on the same device")
|
||||
if not a.is_contiguous() or not b.is_contiguous():
|
||||
raise ValueError("a and b must be contiguous")
|
||||
if not b.is_contiguous():
|
||||
raise ValueError("b must be contiguous")
|
||||
if a.shape[1] != b.shape[1]:
|
||||
raise ValueError("a and b must have matching K dimensions")
|
||||
if not 1 <= a.shape[0] <= 16:
|
||||
@@ -236,6 +250,8 @@ class ShapeDynamicSkinnyGemm:
|
||||
)
|
||||
if config.static_k is not None and a.shape[1] != config.static_k:
|
||||
raise ValueError("input K must match config static_k")
|
||||
if not row_stride_ok(a, config):
|
||||
raise ValueError("a must be row-major with vector_width-aligned rows")
|
||||
has_residual = residual is not None
|
||||
cache_key = (a.dtype, config, has_residual)
|
||||
if cache_key not in self._compiled:
|
||||
|
||||
@@ -13,6 +13,7 @@ from torch import nn
|
||||
import vllm.envs as envs
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.skinny_gemm import (
|
||||
SkinnyGemmConfig,
|
||||
row_stride_ok,
|
||||
shape_dynamic_skinny_gemm,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
|
||||
@@ -304,10 +305,13 @@ def _is_packed_row_major(tensor: torch.Tensor) -> bool:
|
||||
return tensor.dim() == 2 and tensor.stride() == (tensor.shape[1], 1)
|
||||
|
||||
|
||||
def _runtime_ok(x: torch.Tensor, weight: torch.Tensor) -> bool:
|
||||
def _runtime_ok(
|
||||
x: torch.Tensor, weight: torch.Tensor, config: SkinnyGemmConfig
|
||||
) -> bool:
|
||||
return (
|
||||
not envs.VLLM_BATCH_INVARIANT
|
||||
and _is_packed_row_major(x)
|
||||
and x.dim() == 2
|
||||
and row_stride_ok(x, config)
|
||||
and _is_packed_row_major(weight)
|
||||
and x.dtype == torch.bfloat16
|
||||
and weight.dtype == torch.bfloat16
|
||||
@@ -348,7 +352,7 @@ def _qwen4_exp_low_latency_gemm(x: torch.Tensor, weight: torch.Tensor) -> torch.
|
||||
config = None if plan is None else plan.get(x.shape[0])
|
||||
if (
|
||||
config is not None
|
||||
and _runtime_ok(x, weight)
|
||||
and _runtime_ok(x, weight, config)
|
||||
and shape_dynamic_skinny_gemm.is_available()
|
||||
):
|
||||
return shape_dynamic_skinny_gemm(x, weight, config)
|
||||
|
||||
Reference in New Issue
Block a user