[Core][Feat] Pluggable KVCacheConfigBuilder for platform/model-specific KV cache planning (#53558)
Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Claude Opus 5.5
Yifan Qiao
parent
2268d83bd9
commit
b544587408
@@ -50,6 +50,7 @@ from .interfaces import (
|
||||
is_hybrid,
|
||||
requires_raw_input_tokens,
|
||||
supports_hc_sp,
|
||||
supports_kv_cache_grouping,
|
||||
supports_multimodal,
|
||||
supports_multimodal_encoder_tp_data,
|
||||
supports_multimodal_raw_input_only,
|
||||
@@ -882,6 +883,7 @@ class _ModelInfo:
|
||||
supports_replayssm: bool
|
||||
supports_transcription: bool
|
||||
supports_transcription_only: bool
|
||||
supports_kv_cache_grouping: bool
|
||||
supported_video_pruning_methods: tuple[str, ...]
|
||||
supports_mm_device_do_normalize: bool
|
||||
|
||||
@@ -913,6 +915,7 @@ class _ModelInfo:
|
||||
supports_transcription_only=(
|
||||
supports_transcription(model) and model.supports_transcription_only
|
||||
),
|
||||
supports_kv_cache_grouping=supports_kv_cache_grouping(model),
|
||||
has_noops=has_noops(model),
|
||||
supported_video_pruning_methods=getattr(
|
||||
model, "supported_video_pruning_methods", ()
|
||||
|
||||
Reference in New Issue
Block a user