From b68d7ef2622d2d22e964dd842381021865e942b8 Mon Sep 17 00:00:00 2001 From: Jonguk Cheong Date: Mon, 27 Jul 2026 02:59:09 +0900 Subject: [PATCH] [Bugfix][KV Offload] Namespace auto cache dtype by effective dtype (#49438) Signed-off-by: Jonguk Cheong Co-authored-by: OpenAI Codex Co-authored-by: Or Ozeri --- .../kv_transfer/kv_connector/v1/offloading/config.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py index 2a5659c2a9..b86ebf96bb 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py @@ -138,6 +138,12 @@ def build_offloading_config( ) kv_events_config = vllm_config.kv_events_config + cache_dtype = ( + vllm_config.model_config.dtype + if vllm_config.cache_config.cache_dtype == "auto" + else vllm_config.cache_config.cache_dtype + ) + return OffloadingConfig( groups=groups, worker_kv_bytes_per_block=worker_kv_bytes_per_block, @@ -148,7 +154,7 @@ def build_offloading_config( engine_id=engine_id, model=OffloadingModelConfig( name=vllm_config.model_config.model, - dtype=str(vllm_config.cache_config.cache_dtype).replace("torch.", ""), + dtype=str(cache_dtype).removeprefix("torch."), ), cache=OffloadingCacheConfig( tokens_per_hash=tokens_per_hash,