[Model Runner v2] Support update_config (#42783)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
1ac10f159a
commit
f5d3dc7115
@@ -337,6 +337,17 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
self.reset_encoder_cache()
|
self.reset_encoder_cache()
|
||||||
self.reset_mm_cache()
|
self.reset_mm_cache()
|
||||||
|
|
||||||
|
def update_config(self, *args, **kwargs) -> None:
|
||||||
|
# TODO(Wentao): Use full version instead of import when fully migrated to v2
|
||||||
|
from vllm.v1.worker.gpu_model_runner import GPUModelRunner as GPUModelRunnerV1
|
||||||
|
|
||||||
|
GPUModelRunnerV1.update_config(self, *args, **kwargs) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
# v2 reads config via self.vllm_config (e.g. in load_model), so keep it
|
||||||
|
# in sync with the attributes the v1 helper just replaced.
|
||||||
|
self.vllm_config.model_config = self.model_config
|
||||||
|
self.vllm_config.load_config = self.load_config
|
||||||
|
|
||||||
@functools.cached_property
|
@functools.cached_property
|
||||||
def main_stream(self) -> torch.cuda.Stream:
|
def main_stream(self) -> torch.cuda.Stream:
|
||||||
# Cache the default CUDA stream to avoid lookup overhead.
|
# Cache the default CUDA stream to avoid lookup overhead.
|
||||||
|
|||||||
Reference in New Issue
Block a user