[Model] add GLM-5.3-Flash support (#53906)
This commit is contained in:
+37
-7
@@ -86,6 +86,7 @@ def _get_backend_priorities(
|
||||
num_heads: int | None = None,
|
||||
kv_cache_dtype: CacheDType | None = None,
|
||||
use_non_causal: bool = False,
|
||||
head_size: int | None = None,
|
||||
) -> list[AttentionBackendEnum]:
|
||||
"""Get backend priorities with lazy import to avoid circular dependency."""
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
@@ -133,13 +134,21 @@ def _get_backend_priorities(
|
||||
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM120,
|
||||
]
|
||||
else:
|
||||
sparse_tail = [
|
||||
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE,
|
||||
]
|
||||
flashinfer_sparse = AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM90
|
||||
if head_size == 512:
|
||||
sparse_tail.insert(0, flashinfer_sparse)
|
||||
else:
|
||||
sparse_tail.append(flashinfer_sparse)
|
||||
return [
|
||||
AttentionBackendEnum.FLASH_ATTN_MLA,
|
||||
AttentionBackendEnum.FLASHMLA,
|
||||
AttentionBackendEnum.FLASHINFER_MLA,
|
||||
AttentionBackendEnum.TRITON_MLA,
|
||||
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE,
|
||||
*sparse_tail,
|
||||
]
|
||||
else:
|
||||
# SM100f defaults to FlashInfer for TRTLLM causal attention, but its non-causal
|
||||
@@ -373,11 +382,12 @@ class CudaPlatformBase(Platform):
|
||||
invalid_reasons: dict[AttentionBackendEnum, tuple[int, list[str]]] = {}
|
||||
|
||||
backend_priorities = _get_backend_priorities(
|
||||
attn_selector_config.use_mla,
|
||||
device_capability,
|
||||
num_heads,
|
||||
attn_selector_config.kv_cache_dtype,
|
||||
attn_selector_config.use_non_causal,
|
||||
use_mla=attn_selector_config.use_mla,
|
||||
device_capability=device_capability,
|
||||
num_heads=num_heads,
|
||||
kv_cache_dtype=attn_selector_config.kv_cache_dtype,
|
||||
use_non_causal=attn_selector_config.use_non_causal,
|
||||
head_size=attn_selector_config.head_size,
|
||||
)
|
||||
for priority, backend in enumerate(backend_priorities):
|
||||
try:
|
||||
@@ -400,6 +410,26 @@ class CudaPlatformBase(Platform):
|
||||
|
||||
return valid_backends_priorities, invalid_reasons
|
||||
|
||||
@classmethod
|
||||
def _get_indexer_block_alignment(cls, vllm_config: VllmConfig) -> int | None:
|
||||
index_kpool = getattr(
|
||||
vllm_config.model_config.hf_text_config, "index_kpool", None
|
||||
)
|
||||
if not index_kpool or index_kpool <= 1:
|
||||
return None
|
||||
from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
|
||||
|
||||
# kpool paged-MQA indexer: the storage block (block_size /
|
||||
# index_kpool) is virtually split into pool pages, so block_size
|
||||
# must be a multiple of index_kpool times a legal pool page.
|
||||
page = min(PAGED_MQA_PAGE_SIZES)
|
||||
if cls.is_device_capability_family(120):
|
||||
# On sm120 the DeepGEMM paged-MQA kernel only accepts block_kv
|
||||
# 64 for the fp8 indexer cache, so align to the largest pool
|
||||
# page here to make the page split land on 64 not the min 32.
|
||||
page = max(PAGED_MQA_PAGE_SIZES)
|
||||
return index_kpool * page
|
||||
|
||||
@classmethod
|
||||
def get_attn_backend_cls(
|
||||
cls,
|
||||
|
||||
@@ -763,6 +763,17 @@ class Platform:
|
||||
if cache_config.mamba_page_size_padded is not None:
|
||||
cache_config.mamba_page_size_padded = shared_page
|
||||
|
||||
@classmethod
|
||||
def _get_indexer_block_alignment(cls, vllm_config: "VllmConfig") -> int | None:
|
||||
"""Extra ``block_size`` multiple a sparse indexer needs, else ``None``.
|
||||
|
||||
The CUDA kpool paged-MQA indexer virtually splits each storage block
|
||||
into pool pages, so ``block_size`` must be a multiple of
|
||||
``index_kpool * min(PAGED_MQA_PAGE_SIZES)`` — implemented in the CUDA
|
||||
platform override. Other platforms impose no extra constraint.
|
||||
"""
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _align_hybrid_block_size(
|
||||
cls,
|
||||
@@ -805,6 +816,7 @@ class Platform:
|
||||
num_kv_heads=model_config.get_num_kv_heads(parallel_config),
|
||||
head_size=model_config.get_head_size(),
|
||||
dtype=kv_cache_dtype,
|
||||
cache_dtype_str=cache_config.cache_dtype,
|
||||
kv_quant_mode=kv_quant_mode,
|
||||
).page_size_bytes
|
||||
elif cache_config.cache_dtype.startswith("turboquant_"):
|
||||
@@ -912,6 +924,9 @@ class Platform:
|
||||
mamba_page_size,
|
||||
kernel_block_alignment_size * attn_page_size_1_token,
|
||||
)
|
||||
indexer_align = cls._get_indexer_block_alignment(vllm_config)
|
||||
if indexer_align:
|
||||
attn_block_size = indexer_align * cdiv(attn_block_size, indexer_align)
|
||||
|
||||
if cache_config.block_size < attn_block_size:
|
||||
cache_config.block_size = attn_block_size
|
||||
|
||||
Reference in New Issue
Block a user