[Model] add GLM-5.3-Flash support (#53906)
This commit is contained in:
@@ -29,6 +29,11 @@ _DEEPGEMM_BLACKWELL_EXCLUDED_MODEL_TYPES: set[str] = {
|
||||
"qwen3_5_moe_text",
|
||||
}
|
||||
|
||||
# KV page sizes (in cache entries) supported by the paged-MQA logits kernels
|
||||
# (fp8_fp4_paged_mqa_logits / get_paged_mqa_logits_metadata). Larger storage
|
||||
# blocks must be virtually split into one of these page sizes.
|
||||
PAGED_MQA_PAGE_SIZES = (32, 64)
|
||||
|
||||
|
||||
def should_auto_disable_deep_gemm(model_type: str | None) -> bool:
|
||||
"""Check if DeepGemm should be auto-disabled for this model on Blackwell.
|
||||
@@ -648,6 +653,13 @@ def fp8_fp4_paged_mqa_logits(
|
||||
_lazy_init()
|
||||
if _fp8_fp4_paged_mqa_logits_impl is None:
|
||||
return _missing()
|
||||
# DeepGEMM asserts block_tables.stride(-1)==1. A trailing size-1 dim
|
||||
# (e.g. block_table shape [B,1] for short seqs under a large block_size)
|
||||
# can be a transposed view where .contiguous() is a no-op (torch treats the
|
||||
# size-1 dim's stride as irrelevant) yet stride(-1)!=1, failing the kernel.
|
||||
# clone to contiguous format to force stride(-1)==1.
|
||||
if block_tables.dim() >= 2 and block_tables.stride(-1) != 1:
|
||||
block_tables = block_tables.clone(memory_format=torch.contiguous_format)
|
||||
kwargs = {} if indices is None else {"indices": indices}
|
||||
return _fp8_fp4_paged_mqa_logits_impl(
|
||||
q,
|
||||
|
||||
@@ -292,6 +292,32 @@ def has_flashinfer_moe() -> bool:
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def has_flashinfer_sm90_nope_mla() -> bool:
|
||||
"""FlashInfer SM90 NoPE MLA (FP8 KV with in-kernel dequant, kpe=0).
|
||||
|
||||
Feature-detected via the ``ckv_scale_arr`` run() kwarg introduced with
|
||||
the SM90 NoPE support (FlashInfer >= 0.6.18), so dev builds carry the
|
||||
gate without a version parse.
|
||||
"""
|
||||
if not has_flashinfer():
|
||||
return False
|
||||
try:
|
||||
import inspect
|
||||
|
||||
from flashinfer.mla import BatchMLAPagedAttentionWrapper
|
||||
except ImportError:
|
||||
return False
|
||||
try:
|
||||
params = inspect.signature(BatchMLAPagedAttentionWrapper.run).parameters
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return (
|
||||
"ckv_scale_arr" in params
|
||||
and params["ckv_scale_arr"].kind is inspect.Parameter.KEYWORD_ONLY
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def has_flashinfer_sparse_mla_sm120() -> bool:
|
||||
"""Return ``True`` if FlashInfer sparse MLA decode support is available."""
|
||||
|
||||
Reference in New Issue
Block a user