[KV Connector] Support disk offloading in MooncakeStoreConnector (#42689)
Signed-off-by: Zhewen Li <zhewenli@inferact.ai> Co-authored-by: Zhewen Li <zhewenli@inferact.ai> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Zhewen Li
Claude Opus 4.7
parent
787bc0d031
commit
36e74c9ea4
@@ -191,6 +191,10 @@ if TYPE_CHECKING:
|
||||
VLLM_NIXL_SIDE_CHANNEL_HOST: str = "localhost"
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT: int = 5600
|
||||
VLLM_MOONCAKE_BOOTSTRAP_PORT: int = 8998
|
||||
VLLM_MOONCAKE_STORE_TIER_LOG: bool = False
|
||||
VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO: float = 0.9
|
||||
MOONCAKE_PREFERRED_SEGMENT: str | None = None
|
||||
MOONCAKE_REQUESTER_LOCAL_HOSTNAME: str | None = None
|
||||
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840
|
||||
VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
|
||||
VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16
|
||||
@@ -1389,6 +1393,20 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_MOONCAKE_BOOTSTRAP_PORT": lambda: int(
|
||||
os.getenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", "8998")
|
||||
),
|
||||
# Log per-batch memory/disk tier breakdown on external GETs.
|
||||
"VLLM_MOONCAKE_STORE_TIER_LOG": lambda: (
|
||||
os.getenv("VLLM_MOONCAKE_STORE_TIER_LOG", "False").lower() in ("true", "1")
|
||||
),
|
||||
# Fraction of the owner's DirectIO staging buffer to fill per GET batch.
|
||||
"VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO": lambda: float(
|
||||
os.getenv("VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO", "0.9")
|
||||
),
|
||||
# Pin this rank to a specific owner segment ("host:port").
|
||||
"MOONCAKE_PREFERRED_SEGMENT": lambda: os.getenv("MOONCAKE_PREFERRED_SEGMENT"),
|
||||
# Override the hostname the rank registers as a Mooncake requester.
|
||||
"MOONCAKE_REQUESTER_LOCAL_HOSTNAME": lambda: os.getenv(
|
||||
"MOONCAKE_REQUESTER_LOCAL_HOSTNAME"
|
||||
),
|
||||
# Flashinfer MoE backend for vLLM's fused Mixture-of-Experts support.
|
||||
# Both require compute capability 10.0 or above.
|
||||
# Available options:
|
||||
|
||||
Reference in New Issue
Block a user