[KV Offload] Pass OffloadingSpec instead of VllmConfig to secondary tiers (#43076)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
Ronen Schaffer
2026-05-20 03:32:08 +00:00
committed by GitHub
parent cd0ff26e7a
commit 4f940896a3
5 changed files with 15 additions and 15 deletions
@@ -29,7 +29,7 @@ from vllm.v1.kv_offload.tiering.manager import (
)
_CTX = ReqContext(req_id="test")
_MOCK_VLLM_CONFIG = MagicMock()
_MOCK_OFFLOADING_SPEC = MagicMock()
def _mock_mmap_region(num_blocks: int, row_bytes: int = 16):
@@ -68,7 +68,7 @@ class TestExampleSecondaryTierManager:
"""Test basic store and lookup operations."""
mock_view = memoryview(torch.zeros((10, 16), dtype=torch.int8).numpy())
tier = ExampleSecondaryTierManager(
vllm_config=_MOCK_VLLM_CONFIG,
offloading_spec=_MOCK_OFFLOADING_SPEC,
primary_kv_view=mock_view,
tier_type="example",
custom_param=67,
@@ -105,12 +105,12 @@ class TestTieringOffloadingManager:
# Create secondary tiers with the primary view
self.secondary_tier1 = ExampleSecondaryTierManager(
vllm_config=_MOCK_VLLM_CONFIG,
offloading_spec=_MOCK_OFFLOADING_SPEC,
primary_kv_view=mock_view,
tier_type="example",
)
self.secondary_tier2 = ExampleSecondaryTierManager(
vllm_config=_MOCK_VLLM_CONFIG,
offloading_spec=_MOCK_OFFLOADING_SPEC,
primary_kv_view=mock_view,
tier_type="example",
)
+4 -4
View File
@@ -14,7 +14,7 @@ import numpy as np
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.kv_offload.base import OffloadingSpec
# Type alias for job IDs used in async transfer tracking
JobId = int
@@ -55,18 +55,18 @@ class SecondaryTierManager(ABC):
def __init__(
self,
vllm_config: "VllmConfig",
offloading_spec: "OffloadingSpec",
primary_kv_view: memoryview,
tier_type: str,
) -> None:
"""
Args:
vllm_config: Global vLLM configuration.
offloading_spec: Offloading configuration.
primary_kv_view: Memoryview of the primary tier's CPU KV cache.
tier_type: Tier type identifier, set by SecondaryTierFactory
from the registered tier type.
"""
self._vllm_config = vllm_config
self._offloading_spec = offloading_spec
self._primary_kv_view: memoryview = primary_kv_view
self.tier_type = tier_type
@@ -23,7 +23,7 @@ from vllm.v1.kv_offload.tiering.base import (
logger = logging.getLogger(__name__)
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.kv_offload.base import OffloadingSpec
class ExampleSecondaryTierManager(SecondaryTierManager):
@@ -37,7 +37,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
def __init__(
self,
vllm_config: "VllmConfig",
offloading_spec: "OffloadingSpec",
primary_kv_view: memoryview,
tier_type: str,
custom_param: int = 0,
@@ -49,7 +49,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
custom_param: Dummy parameter demonstrating custom args.
"""
super().__init__(
vllm_config=vllm_config,
offloading_spec=offloading_spec,
primary_kv_view=primary_kv_view,
tier_type=tier_type,
)
+3 -3
View File
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING
from vllm.v1.kv_offload.tiering.base import SecondaryTierManager
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.kv_offload.base import OffloadingSpec
class SecondaryTierFactory:
@@ -29,7 +29,7 @@ class SecondaryTierFactory:
cls,
tier_config: dict,
primary_kv_view: memoryview,
vllm_config: "VllmConfig",
offloading_spec: "OffloadingSpec",
) -> SecondaryTierManager:
config = tier_config.copy()
@@ -45,7 +45,7 @@ class SecondaryTierFactory:
tier_cls = cls._registry[tier_type]()
return tier_cls(
vllm_config=vllm_config,
offloading_spec=offloading_spec,
primary_kv_view=primary_kv_view,
tier_type=tier_type,
**config,
+1 -1
View File
@@ -124,7 +124,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
for i, tier_config in enumerate(self.secondary_tier_configs):
try:
tier = SecondaryTierFactory.create_secondary_tier(
tier_config, primary_kv_view, self.vllm_config
tier_config, primary_kv_view, self
)
secondary_tiers.append(tier)
logger.info(