[Model] add GLM-5.3-Flash support (#53906)
This commit is contained in:
@@ -158,6 +158,7 @@ class KVCacheSpec:
|
||||
|
||||
@property
|
||||
def prefix_cacheable(self) -> bool:
|
||||
"""Whether this spec's group participates in prefix caching."""
|
||||
return True
|
||||
|
||||
@property
|
||||
@@ -555,6 +556,8 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
# DeepseekV4 only fields. Non-DeepseekV4 MLA models leave these at defaults.
|
||||
alignment: int | None = None # Default to None for no padding.
|
||||
model_version: str | None = None
|
||||
storage_block_size: int | None = None
|
||||
"""Token width used to view storage when it differs from the kernel block."""
|
||||
# Marks draft groups that flatten a non-causal query block into decode rows.
|
||||
non_causal_multi_token_decode: bool = False
|
||||
# MLA stores a single latent vector per state; there is no separate V.
|
||||
@@ -572,13 +575,16 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
cache_dtype_str_set = set(spec.cache_dtype_str for spec in specs)
|
||||
tokens_per_state_set = set(spec.tokens_per_state for spec in specs)
|
||||
model_version_set = set(spec.model_version for spec in specs)
|
||||
storage_block_size_set = set(spec.storage_block_size for spec in specs)
|
||||
assert (
|
||||
len(cache_dtype_str_set) == 1
|
||||
and len(tokens_per_state_set) == 1
|
||||
and len(model_version_set) == 1
|
||||
and len(storage_block_size_set) == 1
|
||||
), (
|
||||
"All attention layers in the same KV cache group must use the same "
|
||||
"quantization method, tokens per state, and model version."
|
||||
"quantization method, tokens per state, model version, and storage "
|
||||
"block size."
|
||||
)
|
||||
non_causal_mtd_set = {spec.non_causal_multi_token_decode for spec in specs}
|
||||
assert len(non_causal_mtd_set) == 1, (
|
||||
@@ -597,6 +603,7 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
cache_dtype_str=cache_dtype_str_set.pop(),
|
||||
tokens_per_state=tokens_per_state_set.pop(),
|
||||
model_version=model_version_set.pop(),
|
||||
storage_block_size=storage_block_size_set.pop(),
|
||||
non_causal_multi_token_decode=non_causal_mtd_set.pop(),
|
||||
)
|
||||
for spec in specs:
|
||||
@@ -853,6 +860,28 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class KpoolTailSpec(SlidingWindowSpec):
|
||||
"""One-block circular scratch cache for a kpool indexer's raw tail."""
|
||||
|
||||
def max_admission_blocks_per_request(
|
||||
self, max_in_flight_tokens: int, max_model_len: int
|
||||
) -> int:
|
||||
return 1
|
||||
|
||||
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
|
||||
return 1
|
||||
|
||||
def is_uniform_with_collection(
|
||||
self, kv_cache_specs: dict[str, KVCacheSpec]
|
||||
) -> bool:
|
||||
return all(isinstance(spec, KpoolTailSpec) for spec in kv_cache_specs.values())
|
||||
|
||||
@property
|
||||
def prefix_cacheable(self) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MambaSpec(KVCacheSpec):
|
||||
shapes: tuple[tuple[int, ...], ...]
|
||||
@@ -875,6 +904,10 @@ class MambaSpec(KVCacheSpec):
|
||||
for (shape, dtype) in zip(self.shapes, self.dtypes)
|
||||
)
|
||||
|
||||
@property
|
||||
def real_page_size_bytes(self) -> int:
|
||||
return self.state_content_size_bytes
|
||||
|
||||
@property
|
||||
def page_size_bytes(self) -> int:
|
||||
page_size = sum(
|
||||
|
||||
Reference in New Issue
Block a user