[Model Runner v2] Fix block table IMA issue (#40648)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -46,7 +46,7 @@ from vllm.tasks import SupportedTask
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.utils.mem_utils import MemorySnapshot, format_gib, memory_profiling
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache, set_random_seed
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
|
||||
from vllm.v1.outputs import (
|
||||
@@ -192,15 +192,8 @@ class Worker(WorkerBase):
|
||||
buffer.data.copy_(self._sleep_saved_buffers[name].data)
|
||||
self._sleep_saved_buffers = {}
|
||||
|
||||
# If the KV cache has just been woken up,
|
||||
# the internal state of cache_engine must be reset,
|
||||
# especially the FP8 scaling factor.
|
||||
if (
|
||||
(tags is None or "kv_cache" in tags)
|
||||
and is_quantized_kv_cache(self.cache_config.cache_dtype)
|
||||
and hasattr(self.model_runner, "init_fp8_kv_scales")
|
||||
):
|
||||
self.model_runner.init_fp8_kv_scales()
|
||||
if tags is None or "kv_cache" in tags:
|
||||
self.model_runner.post_kv_cache_wake_up()
|
||||
|
||||
def _maybe_get_memory_pool_context(self, tag: str) -> AbstractContextManager:
|
||||
if not self.vllm_config.model_config.enable_sleep_mode:
|
||||
|
||||
Reference in New Issue
Block a user