Enable DeepSeek V4 and GLM-5.1 on SM120 (#43477)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Yongye Zhu
parent
3ce15fd574
commit
44d95069e9
@@ -155,17 +155,17 @@ class CompressorStateCache(torch.nn.Module, AttentionLayerBase):
|
||||
raise ValueError(f"Invalid compress ratio: {compress_ratio}")
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
|
||||
# FlashMLA's UE8M0 paged layout needs 576B alignment; the FlashInfer
|
||||
# full-cache path shares state pages with contiguous KV pages, so
|
||||
# padding would break page matching.
|
||||
is_flashmla = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
|
||||
# fp8_ds_mla is the UE8M0 paged layout and needs 576B alignment. Plain
|
||||
# full-cache rows share state pages with contiguous KV pages, so padding
|
||||
# would break page matching.
|
||||
uses_fp8_ds_mla_layout = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
|
||||
return SlidingWindowMLASpec( # only has one vector instead of K + V
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=self.state_dim,
|
||||
dtype=self.dtype,
|
||||
sliding_window=self.sliding_window,
|
||||
alignment=576 if is_flashmla else None,
|
||||
alignment=576 if uses_fp8_ds_mla_layout else None,
|
||||
)
|
||||
|
||||
def forward(self): ...
|
||||
@@ -340,8 +340,8 @@ class DeepseekCompressor(nn.Module):
|
||||
k_cache_layer = self._static_forward_context[self.k_cache_prefix]
|
||||
kv_cache = k_cache_layer.kv_cache
|
||||
|
||||
# FlashInfer V4 reads a contiguous bf16 / per-tensor fp8 cache row; the
|
||||
# legacy FlashMLA path uses the UE8M0 paged uint8 layout.
|
||||
# Plain-row V4 reads a contiguous bf16 / per-tensor fp8 cache row; the
|
||||
# fp8_ds_mla path uses the UE8M0 paged uint8 layout.
|
||||
store_full_kv = self.head_dim == 512 and kv_cache.dtype != torch.uint8
|
||||
store_full_fp8 = kv_cache.dtype == torch.float8_e4m3fn
|
||||
fp8_scale = (
|
||||
@@ -358,8 +358,8 @@ class DeepseekCompressor(nn.Module):
|
||||
compress_norm_rope_store_cutedsl,
|
||||
)
|
||||
|
||||
# head=512 on CUDA always uses cutedsl, for both the legacy UE8M0
|
||||
# layout and the FlashInfer full-cache layout. The full-cache flags
|
||||
# head=512 on CUDA always uses cutedsl, for both the fp8_ds_mla
|
||||
# layout and the plain full-cache layout. The full-cache flags
|
||||
# are consumed only here.
|
||||
compress_norm_rope_store_fn = compress_norm_rope_store_cutedsl
|
||||
extra_kwargs: dict[str, Any] = dict(
|
||||
|
||||
Reference in New Issue
Block a user