[Feature] Support batch invariance on ROCm (#52231)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Andreas Karatzas
parent
50b21a2667
commit
e0317b28ce
@@ -62,7 +62,12 @@ if is_aiter_found_and_supported():
|
||||
N = weight.shape[0]
|
||||
K = weight.shape[1]
|
||||
if rocm_use_aiter_fp4_asm_gemm:
|
||||
if M <= 64 and rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K):
|
||||
# The preshuffle kernel's tuned configs split K in some M buckets.
|
||||
if (
|
||||
M <= 64
|
||||
and not envs.VLLM_BATCH_INVARIANT
|
||||
and rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K)
|
||||
):
|
||||
if x_scales is None:
|
||||
# use hip quant kernel for performance
|
||||
if M >= 32:
|
||||
@@ -116,7 +121,16 @@ if is_aiter_found_and_supported():
|
||||
x_q.shape[0], weight.shape[0], device=x_q.device, dtype=out_dtype
|
||||
)
|
||||
|
||||
gemm_afp4wfp4(x_q, weight, x_s, weight_scale.T, out_dtype, y)
|
||||
config = None
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
# The configs split K in some M buckets (up to M <= 256 for
|
||||
# tuned shapes). Keep the tile, drop the split.
|
||||
from aiter.ops.triton.gemm_afp4wfp4 import _get_config
|
||||
|
||||
config, _ = _get_config(M, N, K)
|
||||
config = dict(config, NUM_KSPLIT=1)
|
||||
|
||||
gemm_afp4wfp4(x_q, weight, x_s, weight_scale.T, out_dtype, y, config)
|
||||
return y
|
||||
|
||||
def gemm_with_dynamic_quant_fake(
|
||||
|
||||
Reference in New Issue
Block a user