[PD] Change kv_load_failure_policy Default from "recompute" to "fail" (#34896)
Signed-off-by: NickLucche <nlucches@redhat.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
8dc8a99b56
commit
ab6f3487a6
@@ -30,7 +30,7 @@ def _make_get_num_new_matched_tokens(
|
||||
|
||||
@pytest.fixture
|
||||
def scheduler():
|
||||
vllm_config = create_vllm_config()
|
||||
vllm_config = create_vllm_config(kv_load_failure_policy="recompute")
|
||||
return create_scheduler(vllm_config)
|
||||
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain, count
|
||||
from typing import Any
|
||||
from typing import Any, Literal
|
||||
|
||||
import torch
|
||||
|
||||
@@ -96,6 +96,7 @@ def create_vllm_config(
|
||||
cache_dtype: str = "auto",
|
||||
hf_overrides: dict[str, Any] | None = None,
|
||||
attention_backend: str | None = None,
|
||||
kv_load_failure_policy: Literal["recompute", "fail"] = "fail",
|
||||
) -> VllmConfig:
|
||||
"""Initialize VllmConfig For Testing."""
|
||||
model_config = ModelConfig(
|
||||
@@ -125,6 +126,7 @@ def create_vllm_config(
|
||||
kv_role="kv_both",
|
||||
enable_permute_local_kv=enable_permute_local_kv,
|
||||
kv_connector_extra_config=kv_connector_extra_config or {},
|
||||
kv_load_failure_policy=kv_load_failure_policy,
|
||||
)
|
||||
attention_config = AttentionConfig(backend=attention_backend)
|
||||
return VllmConfig(
|
||||
|
||||
Reference in New Issue
Block a user