[Core] Add update_config RPC method (#20095)
Signed-off-by: 22quinn <33176974+22quinn@users.noreply.github.com>
This commit is contained in:
@@ -19,7 +19,7 @@ from vllm.attention.backends.abstract import AttentionBackend
|
||||
from vllm.attention.layer import Attention
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.config import (CompilationLevel, VllmConfig,
|
||||
get_layers_from_vllm_config)
|
||||
get_layers_from_vllm_config, update_config)
|
||||
from vllm.distributed.eplb.eplb_state import EplbState
|
||||
from vllm.distributed.kv_transfer import (get_kv_transfer_group,
|
||||
has_kv_transfer_group)
|
||||
@@ -1728,6 +1728,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
draft_token_ids.append(drafter_output.tolist())
|
||||
return draft_token_ids
|
||||
|
||||
def update_config(self, overrides: dict[str, Any]) -> None:
|
||||
allowed_config_names = {"load_config", "model_config"}
|
||||
for config_name, config_overrides in overrides.items():
|
||||
assert config_name in allowed_config_names, \
|
||||
f"Config `{config_name}` not supported. " \
|
||||
f"Allowed configs: {allowed_config_names}"
|
||||
config = getattr(self, config_name)
|
||||
new_config = update_config(config, config_overrides)
|
||||
setattr(self, config_name, new_config)
|
||||
|
||||
def load_model(self) -> None:
|
||||
logger.info("Starting to load model %s...", self.model_config.model)
|
||||
with DeviceMemoryProfiler() as m: # noqa: SIM117
|
||||
|
||||
Reference in New Issue
Block a user