[KV Connector] Support disk offloading in MooncakeStoreConnector (#42689)

Signed-off-by: Zhewen Li <zhewenli@inferact.ai>
Co-authored-by: Zhewen Li <zhewenli@inferact.ai>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Zhewen Li
2026-05-16 13:34:15 -07:00
committed by GitHub
co-authored by Zhewen Li Claude Opus 4.7
parent 787bc0d031
commit 36e74c9ea4
5 changed files with 1385 additions and 55 deletions
+18
View File
@@ -191,6 +191,10 @@ if TYPE_CHECKING:
VLLM_NIXL_SIDE_CHANNEL_HOST: str = "localhost"
VLLM_NIXL_SIDE_CHANNEL_PORT: int = 5600
VLLM_MOONCAKE_BOOTSTRAP_PORT: int = 8998
VLLM_MOONCAKE_STORE_TIER_LOG: bool = False
VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO: float = 0.9
MOONCAKE_PREFERRED_SEGMENT: str | None = None
MOONCAKE_REQUESTER_LOCAL_HOSTNAME: str | None = None
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840
VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16
@@ -1389,6 +1393,20 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_MOONCAKE_BOOTSTRAP_PORT": lambda: int(
os.getenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", "8998")
),
# Log per-batch memory/disk tier breakdown on external GETs.
"VLLM_MOONCAKE_STORE_TIER_LOG": lambda: (
os.getenv("VLLM_MOONCAKE_STORE_TIER_LOG", "False").lower() in ("true", "1")
),
# Fraction of the owner's DirectIO staging buffer to fill per GET batch.
"VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO": lambda: float(
os.getenv("VLLM_MOONCAKE_DISK_STAGING_USABLE_RATIO", "0.9")
),
# Pin this rank to a specific owner segment ("host:port").
"MOONCAKE_PREFERRED_SEGMENT": lambda: os.getenv("MOONCAKE_PREFERRED_SEGMENT"),
# Override the hostname the rank registers as a Mooncake requester.
"MOONCAKE_REQUESTER_LOCAL_HOSTNAME": lambda: os.getenv(
"MOONCAKE_REQUESTER_LOCAL_HOSTNAME"
),
# Flashinfer MoE backend for vLLM's fused Mixture-of-Experts support.
# Both require compute capability 10.0 or above.
# Available options: