[UX] Add model_class_overrides for development and debugging (#47148)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
c64c356990
commit
32ab064621
@@ -74,6 +74,12 @@ else:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Process-local record of which (arch, target) model-class overrides have been
|
||||
# registered in *this* process. Must not live on ModelConfig: that instance is
|
||||
# pickled to each worker, so an instance flag would arrive already "registered"
|
||||
# while the worker's own global ModelRegistry is still untouched.
|
||||
_REGISTERED_MODEL_CLASS_OVERRIDES: set[tuple[str, str]] = set()
|
||||
|
||||
RunnerOption = Literal["auto", RunnerType]
|
||||
ConvertType = Literal["none", "embed", "classify"]
|
||||
ConvertOption = Literal["auto", ConvertType]
|
||||
@@ -274,6 +280,13 @@ class ModelConfig:
|
||||
hf_overrides: HfOverrides = field(default_factory=dict)
|
||||
"""If a dictionary, contains arguments to be forwarded to the Hugging Face
|
||||
config. If a callable, it is called to update the HuggingFace config."""
|
||||
model_class_overrides: dict[str, str] = field(default_factory=dict)
|
||||
"""Override the model class used for one or more architectures, mapping the
|
||||
architecture name to a `"module:class"` target (the same format accepted by
|
||||
`ModelRegistry.register_model`). This registers the target class at runtime,
|
||||
e.g. `{"GlmMoeDsaForCausalLM":
|
||||
"vllm.models.deepseek_v32.nvidia.model:DeepseekV32ForCausalLM"}`. This
|
||||
argument is for development and debugging purposes only."""
|
||||
generation_config: str = "auto"
|
||||
"""The folder path to the generation config. Defaults to `"auto"`, the
|
||||
generation config will be loaded from model path. If set to `"vllm"`, no
|
||||
@@ -812,8 +825,34 @@ class ModelConfig:
|
||||
|
||||
@property
|
||||
def registry(self):
|
||||
self._maybe_register_model_class_overrides()
|
||||
return me_models.ModelRegistry
|
||||
|
||||
def _maybe_register_model_class_overrides(self) -> None:
|
||||
# Apply ``model_class_overrides`` here because this property is the
|
||||
# single chokepoint through which every model-class inspect/resolve
|
||||
# passes, in both the engine front-end and every worker process. The
|
||||
# guard is process-local (see ``_REGISTERED_MODEL_CLASS_OVERRIDES``), so
|
||||
# each worker re-registers into its own ModelRegistry exactly once
|
||||
# rather than trusting a pickled-in instance flag.
|
||||
if not self.model_class_overrides:
|
||||
return
|
||||
pending = [
|
||||
(arch, target)
|
||||
for arch, target in self.model_class_overrides.items()
|
||||
if (arch, target) not in _REGISTERED_MODEL_CLASS_OVERRIDES
|
||||
]
|
||||
if not pending:
|
||||
return
|
||||
logger.warning_once(
|
||||
"Applying model_class_overrides %s. This is intended for "
|
||||
"development/debugging.",
|
||||
str(self.model_class_overrides),
|
||||
)
|
||||
for arch, target in pending:
|
||||
me_models.ModelRegistry.register_model(arch, target)
|
||||
_REGISTERED_MODEL_CLASS_OVERRIDES.add((arch, target))
|
||||
|
||||
@property
|
||||
def architectures(self) -> list[str]:
|
||||
return self.model_arch_config.architectures
|
||||
|
||||
Reference in New Issue
Block a user