[Bugfix][KV Offload] Namespace auto cache dtype by effective dtype (#49438)
Signed-off-by: Jonguk Cheong <jdal3031@snu.ac.kr> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
OpenAI Codex
Or Ozeri
parent
7154856f3d
commit
b68d7ef262
@@ -138,6 +138,12 @@ def build_offloading_config(
|
||||
)
|
||||
|
||||
kv_events_config = vllm_config.kv_events_config
|
||||
cache_dtype = (
|
||||
vllm_config.model_config.dtype
|
||||
if vllm_config.cache_config.cache_dtype == "auto"
|
||||
else vllm_config.cache_config.cache_dtype
|
||||
)
|
||||
|
||||
return OffloadingConfig(
|
||||
groups=groups,
|
||||
worker_kv_bytes_per_block=worker_kv_bytes_per_block,
|
||||
@@ -148,7 +154,7 @@ def build_offloading_config(
|
||||
engine_id=engine_id,
|
||||
model=OffloadingModelConfig(
|
||||
name=vllm_config.model_config.model,
|
||||
dtype=str(vllm_config.cache_config.cache_dtype).replace("torch.", ""),
|
||||
dtype=str(cache_dtype).removeprefix("torch."),
|
||||
),
|
||||
cache=OffloadingCacheConfig(
|
||||
tokens_per_hash=tokens_per_hash,
|
||||
|
||||
Reference in New Issue
Block a user