[Bugfix][sleepmode][fp8 kv cache]: Fix FP8 KV cache + sleep(level=2) gibberish output (#28783)
Signed-off-by: vensen <vensenmu@gmail.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
@@ -25,7 +25,7 @@ from vllm.attention.backends.abstract import (
|
||||
AttentionType,
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.attention.layer import Attention
|
||||
from vllm.attention.layer import Attention, MLAAttention
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.compilation.cuda_graph import CUDAGraphWrapper
|
||||
from vllm.compilation.monitor import set_cudagraph_capturing_enabled
|
||||
@@ -602,6 +602,50 @@ class GPUModelRunner(
|
||||
if self.mm_budget:
|
||||
self.mm_budget.reset_cache()
|
||||
|
||||
@torch.inference_mode()
|
||||
def init_fp8_kv_scales(self) -> None:
|
||||
"""
|
||||
Re-initialize the KV cache and FP8 scales after waking from sleep.
|
||||
1. Zero out the KV cache tensors to remove garbage data from re-allocation.
|
||||
2. Reset Attention layer scaling factors (_k_scale, _v_scale) to 1.0.
|
||||
If these are left at 0.0 (default after wake_up), all KV cache values
|
||||
become effectively zero, causing gibberish output.
|
||||
"""
|
||||
if not self.cache_config.cache_dtype.startswith("fp8"):
|
||||
return
|
||||
|
||||
kv_caches = getattr(self, "kv_caches", [])
|
||||
for cache_tensor in kv_caches:
|
||||
if cache_tensor is not None:
|
||||
cache_tensor.zero_()
|
||||
|
||||
k_attr_names = ("_k_scale", "k_scale")
|
||||
v_attr_names = ("_v_scale", "v_scale")
|
||||
|
||||
attn_layers = self.compilation_config.static_forward_context
|
||||
for name, module in attn_layers.items():
|
||||
if isinstance(module, (Attention, MLAAttention)):
|
||||
# TODO: Generally, scale is 1.0 if user uses on-the-fly fp8
|
||||
# kvcache quant. However, to get better accuracy, compression
|
||||
# frameworks like llm-compressors allow users to tune the
|
||||
# scale. We may need to restore the specific calibrated scales
|
||||
# here in the future.
|
||||
k_scale_val, v_scale_val = 1.0, 1.0
|
||||
|
||||
# Processing K Scale
|
||||
for attr in k_attr_names:
|
||||
if hasattr(module, attr):
|
||||
param = getattr(module, attr)
|
||||
if isinstance(param, torch.Tensor):
|
||||
param.fill_(k_scale_val)
|
||||
|
||||
# Processing V Scale
|
||||
for attr in v_attr_names:
|
||||
if hasattr(module, attr):
|
||||
param = getattr(module, attr)
|
||||
if isinstance(param, torch.Tensor):
|
||||
param.fill_(v_scale_val)
|
||||
|
||||
def _get_positions(self, num_tokens: Any):
|
||||
if isinstance(num_tokens, int):
|
||||
if self.uses_mrope:
|
||||
|
||||
@@ -141,6 +141,16 @@ class Worker(WorkerBase):
|
||||
buffer.data.copy_(self._sleep_saved_buffers[name].data)
|
||||
self._sleep_saved_buffers = {}
|
||||
|
||||
# If the KV cache has just been woken up,
|
||||
# the internal state of cache_engine must be reset,
|
||||
# especially the FP8 scaling factor.
|
||||
if (
|
||||
(tags is None or "kv_cache" in tags)
|
||||
and self.cache_config.cache_dtype.startswith("fp8")
|
||||
and hasattr(self.model_runner, "init_fp8_kv_scales")
|
||||
):
|
||||
self.model_runner.init_fp8_kv_scales()
|
||||
|
||||
def _maybe_get_memory_pool_context(self, tag: str) -> AbstractContextManager:
|
||||
if self.vllm_config.model_config.enable_sleep_mode:
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
|
||||
Reference in New Issue
Block a user