Log fully resolved pooling config at startup (#48030)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
313d01f507
commit
9182e86971
+13
-1
@@ -20,7 +20,7 @@ from vllm.config.multimodal import (
|
||||
MMTensorIPC,
|
||||
MultiModalConfig,
|
||||
)
|
||||
from vllm.config.pooler import PoolerConfig
|
||||
from vllm.config.pooler import POOLER_CONFIG_LOG_FIELDS, PoolerConfig
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.config.scheduler import RunnerType
|
||||
from vllm.config.utils import config, getattr_iter
|
||||
@@ -640,6 +640,13 @@ class ModelConfig:
|
||||
if self.runner_type == "pooling":
|
||||
if self.pooler_config is None:
|
||||
self.pooler_config = PoolerConfig()
|
||||
pooler_config_sources: dict[str, str] = {}
|
||||
else:
|
||||
pooler_config_sources = {
|
||||
k: "user"
|
||||
for k in POOLER_CONFIG_LOG_FIELDS
|
||||
if getattr(self.pooler_config, k) is not None
|
||||
}
|
||||
|
||||
base_config = get_pooling_config(self.model, self.revision)
|
||||
if base_config is not None:
|
||||
@@ -647,13 +654,18 @@ class ModelConfig:
|
||||
for k, v in base_config.items():
|
||||
if getattr(self.pooler_config, k) is None:
|
||||
setattr(self.pooler_config, k, v)
|
||||
pooler_config_sources[k] = "sentence_transformers"
|
||||
|
||||
default_seq_pooling_type = self._model_info.default_seq_pooling_type
|
||||
if self.pooler_config.seq_pooling_type is None:
|
||||
self.pooler_config.seq_pooling_type = default_seq_pooling_type
|
||||
pooler_config_sources["seq_pooling_type"] = "model_default"
|
||||
default_tok_pooling_type = self._model_info.default_tok_pooling_type
|
||||
if self.pooler_config.tok_pooling_type is None:
|
||||
self.pooler_config.tok_pooling_type = default_tok_pooling_type
|
||||
pooler_config_sources["tok_pooling_type"] = "model_default"
|
||||
pooler_config_sources.setdefault("use_activation", "pooler_default")
|
||||
self._pooler_config_sources = pooler_config_sources
|
||||
|
||||
self.dtype: torch.dtype = _get_and_verify_dtype(
|
||||
self.model,
|
||||
|
||||
@@ -16,6 +16,12 @@ SEQ_POOLING_TYPES: tuple[SequencePoolingType, ...] = get_args(SequencePoolingTyp
|
||||
TokenPoolingType = Literal["ALL", "STEP"]
|
||||
TOK_POOLING_TYPES: tuple[TokenPoolingType, ...] = get_args(TokenPoolingType)
|
||||
|
||||
POOLER_CONFIG_LOG_FIELDS = (
|
||||
"seq_pooling_type",
|
||||
"tok_pooling_type",
|
||||
"use_activation",
|
||||
)
|
||||
|
||||
|
||||
@config
|
||||
class PoolerConfig:
|
||||
|
||||
+59
-1
@@ -22,6 +22,7 @@ import zmq
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import ParallelConfig, VllmConfig
|
||||
from vllm.config.pooler import POOLER_CONFIG_LOG_FIELDS
|
||||
from vllm.distributed import (
|
||||
cleanup_dist_env_and_memory,
|
||||
stateless_destroy_torch_distributed_process_group,
|
||||
@@ -121,6 +122,7 @@ class EngineCore:
|
||||
|
||||
# Setup Model.
|
||||
self.model_executor = executor_class(vllm_config)
|
||||
self._pooler_config_logged = False
|
||||
if executor_fail_callback is not None:
|
||||
self.model_executor.register_failure_callback(executor_fail_callback)
|
||||
|
||||
@@ -348,7 +350,63 @@ class EngineCore:
|
||||
return scheduler_kv_cache_config
|
||||
|
||||
def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
|
||||
return self.model_executor.supported_tasks
|
||||
supported_tasks = self.model_executor.supported_tasks
|
||||
self._log_pooler_config(supported_tasks)
|
||||
return supported_tasks
|
||||
|
||||
def _log_pooler_config(self, supported_tasks: tuple[SupportedTask, ...]) -> None:
|
||||
if self._pooler_config_logged:
|
||||
return
|
||||
|
||||
model_config = self.vllm_config.model_config
|
||||
pooler_config = model_config.pooler_config
|
||||
if (
|
||||
self.vllm_config.parallel_config.data_parallel_rank_local
|
||||
or model_config.runner_type != "pooling"
|
||||
or pooler_config is None
|
||||
):
|
||||
return
|
||||
|
||||
supported_pooling_tasks = tuple(
|
||||
sorted(set(supported_tasks) & set(POOLING_TASKS))
|
||||
)
|
||||
if not supported_pooling_tasks:
|
||||
return
|
||||
|
||||
self._pooler_config_logged = True
|
||||
task_set = set(supported_pooling_tasks)
|
||||
use_activation = pooler_config.use_activation
|
||||
if use_activation is None:
|
||||
use_activation = True
|
||||
sources = getattr(model_config, "_pooler_config_sources", {})
|
||||
pooling_type_field = (
|
||||
"seq_pooling_type"
|
||||
if task_set & {"embed", "classify"}
|
||||
else "tok_pooling_type"
|
||||
)
|
||||
|
||||
def log_field(name: str, field: str) -> str:
|
||||
value = (
|
||||
use_activation
|
||||
if field == "use_activation"
|
||||
else getattr(pooler_config, field)
|
||||
)
|
||||
source = sources.get(field, "unknown")
|
||||
return f"{name}={value}(source={source})"
|
||||
|
||||
log_items = [("pooling_type", pooling_type_field)]
|
||||
log_items.extend(
|
||||
(field, field)
|
||||
for field in POOLER_CONFIG_LOG_FIELDS
|
||||
if field != pooling_type_field
|
||||
)
|
||||
config_fields = ", ".join(log_field(name, field) for name, field in log_items)
|
||||
|
||||
logger.info_once(
|
||||
"Resolved pooling config: %s, supported_tasks=%s",
|
||||
config_fields,
|
||||
supported_pooling_tasks,
|
||||
)
|
||||
|
||||
def get_kv_cache_group_metadata(self) -> list[dict[str, int | str | None]]:
|
||||
"""Return msgspec-serializable metadata for scheduler KV cache groups."""
|
||||
|
||||
Reference in New Issue
Block a user