Improve enable chunked_prefill & prefix_caching logic. (#26623)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Signed-off-by: wang.yuqi <noooop@126.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
wang.yuqi
2025-11-27 22:05:48 -08:00
committed by GitHub
co-authored by Cyrus Leung
parent 37b15e97e8
commit f4b76056ee
11 changed files with 456 additions and 133 deletions
+2 -2
View File
@@ -32,7 +32,7 @@ from vllm.tasks import PoolingTask
from vllm.v1.pool.metadata import PoolingMetadata
from .interfaces import SupportsCrossEncoding, SupportsQuant
from .interfaces_base import default_pooling_type
from .interfaces_base import attn_type, default_pooling_type
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
@@ -432,7 +432,6 @@ class BertModel(nn.Module, SupportsQuant):
return loaded_params
@default_pooling_type("ALL")
class BertPoolingModel(BertModel):
is_pooling_model = True
@@ -864,6 +863,7 @@ class BertForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsQu
)
@attn_type("encoder_only")
@default_pooling_type("ALL")
class BertForTokenClassification(nn.Module):
is_pooling_model = True