Improve enable chunked_prefill & prefix_caching logic. (#26623)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Signed-off-by: wang.yuqi <noooop@126.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
wang.yuqi
2025-11-27 22:05:48 -08:00
committed by GitHub
co-authored by Cyrus Leung
parent 37b15e97e8
commit f4b76056ee
11 changed files with 456 additions and 133 deletions
+13 -2
View File
@@ -17,7 +17,7 @@ from collections.abc import Callable, Set
from dataclasses import asdict, dataclass, field
from functools import lru_cache
from pathlib import Path
from typing import TypeVar
from typing import TYPE_CHECKING, Any, TypeVar
import torch.nn as nn
import transformers
@@ -33,6 +33,14 @@ from vllm.logging_utils import logtime
from vllm.transformers_utils.dynamic_module import try_get_class_from_dynamic_module
from vllm.utils.hashing import safe_hash
if TYPE_CHECKING:
from vllm.config.model import AttnTypeStr
from vllm.config.pooler import PoolingTypeStr
else:
AttnTypeStr = Any
PoolingTypeStr = Any
from .interfaces import (
has_inner_state,
has_noops,
@@ -47,6 +55,7 @@ from .interfaces import (
supports_transcription,
)
from .interfaces_base import (
get_attn_type,
get_default_pooling_type,
is_pooling_model,
is_text_generation_model,
@@ -509,7 +518,8 @@ class _ModelInfo:
architecture: str
is_text_generation_model: bool
is_pooling_model: bool
default_pooling_type: str
attn_type: AttnTypeStr
default_pooling_type: PoolingTypeStr
supports_cross_encoding: bool
supports_multimodal: bool
supports_multimodal_raw_input_only: bool
@@ -530,6 +540,7 @@ class _ModelInfo:
is_text_generation_model=is_text_generation_model(model),
is_pooling_model=is_pooling_model(model),
default_pooling_type=get_default_pooling_type(model),
attn_type=get_attn_type(model),
supports_cross_encoding=supports_cross_encoding(model),
supports_multimodal=supports_multimodal(model),
supports_multimodal_raw_input_only=supports_multimodal_raw_input_only(