[UX] Add model_class_overrides for development and debugging (#47148)

Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jee Jee Li
2026-07-07 11:43:08 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent c64c356990
commit 32ab064621
3 changed files with 70 additions and 0 deletions
+39
View File
@@ -74,6 +74,12 @@ else:
logger = init_logger(__name__)
# Process-local record of which (arch, target) model-class overrides have been
# registered in *this* process. Must not live on ModelConfig: that instance is
# pickled to each worker, so an instance flag would arrive already "registered"
# while the worker's own global ModelRegistry is still untouched.
_REGISTERED_MODEL_CLASS_OVERRIDES: set[tuple[str, str]] = set()
RunnerOption = Literal["auto", RunnerType]
ConvertType = Literal["none", "embed", "classify"]
ConvertOption = Literal["auto", ConvertType]
@@ -274,6 +280,13 @@ class ModelConfig:
hf_overrides: HfOverrides = field(default_factory=dict)
"""If a dictionary, contains arguments to be forwarded to the Hugging Face
config. If a callable, it is called to update the HuggingFace config."""
model_class_overrides: dict[str, str] = field(default_factory=dict)
"""Override the model class used for one or more architectures, mapping the
architecture name to a `"module:class"` target (the same format accepted by
`ModelRegistry.register_model`). This registers the target class at runtime,
e.g. `{"GlmMoeDsaForCausalLM":
"vllm.models.deepseek_v32.nvidia.model:DeepseekV32ForCausalLM"}`. This
argument is for development and debugging purposes only."""
generation_config: str = "auto"
"""The folder path to the generation config. Defaults to `"auto"`, the
generation config will be loaded from model path. If set to `"vllm"`, no
@@ -812,8 +825,34 @@ class ModelConfig:
@property
def registry(self):
self._maybe_register_model_class_overrides()
return me_models.ModelRegistry
def _maybe_register_model_class_overrides(self) -> None:
# Apply ``model_class_overrides`` here because this property is the
# single chokepoint through which every model-class inspect/resolve
# passes, in both the engine front-end and every worker process. The
# guard is process-local (see ``_REGISTERED_MODEL_CLASS_OVERRIDES``), so
# each worker re-registers into its own ModelRegistry exactly once
# rather than trusting a pickled-in instance flag.
if not self.model_class_overrides:
return
pending = [
(arch, target)
for arch, target in self.model_class_overrides.items()
if (arch, target) not in _REGISTERED_MODEL_CLASS_OVERRIDES
]
if not pending:
return
logger.warning_once(
"Applying model_class_overrides %s. This is intended for "
"development/debugging.",
str(self.model_class_overrides),
)
for arch, target in pending:
me_models.ModelRegistry.register_model(arch, target)
_REGISTERED_MODEL_CLASS_OVERRIDES.add((arch, target))
@property
def architectures(self) -> list[str]:
return self.model_arch_config.architectures