[KV Offload] Pass OffloadingSpec instead of VllmConfig to secondary tiers (#43076)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
@@ -29,7 +29,7 @@ from vllm.v1.kv_offload.tiering.manager import (
|
||||
)
|
||||
|
||||
_CTX = ReqContext(req_id="test")
|
||||
_MOCK_VLLM_CONFIG = MagicMock()
|
||||
_MOCK_OFFLOADING_SPEC = MagicMock()
|
||||
|
||||
|
||||
def _mock_mmap_region(num_blocks: int, row_bytes: int = 16):
|
||||
@@ -68,7 +68,7 @@ class TestExampleSecondaryTierManager:
|
||||
"""Test basic store and lookup operations."""
|
||||
mock_view = memoryview(torch.zeros((10, 16), dtype=torch.int8).numpy())
|
||||
tier = ExampleSecondaryTierManager(
|
||||
vllm_config=_MOCK_VLLM_CONFIG,
|
||||
offloading_spec=_MOCK_OFFLOADING_SPEC,
|
||||
primary_kv_view=mock_view,
|
||||
tier_type="example",
|
||||
custom_param=67,
|
||||
@@ -105,12 +105,12 @@ class TestTieringOffloadingManager:
|
||||
|
||||
# Create secondary tiers with the primary view
|
||||
self.secondary_tier1 = ExampleSecondaryTierManager(
|
||||
vllm_config=_MOCK_VLLM_CONFIG,
|
||||
offloading_spec=_MOCK_OFFLOADING_SPEC,
|
||||
primary_kv_view=mock_view,
|
||||
tier_type="example",
|
||||
)
|
||||
self.secondary_tier2 = ExampleSecondaryTierManager(
|
||||
vllm_config=_MOCK_VLLM_CONFIG,
|
||||
offloading_spec=_MOCK_OFFLOADING_SPEC,
|
||||
primary_kv_view=mock_view,
|
||||
tier_type="example",
|
||||
)
|
||||
|
||||
@@ -14,7 +14,7 @@ import numpy as np
|
||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.kv_offload.base import OffloadingSpec
|
||||
|
||||
# Type alias for job IDs used in async transfer tracking
|
||||
JobId = int
|
||||
@@ -55,18 +55,18 @@ class SecondaryTierManager(ABC):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
offloading_spec: "OffloadingSpec",
|
||||
primary_kv_view: memoryview,
|
||||
tier_type: str,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
vllm_config: Global vLLM configuration.
|
||||
offloading_spec: Offloading configuration.
|
||||
primary_kv_view: Memoryview of the primary tier's CPU KV cache.
|
||||
tier_type: Tier type identifier, set by SecondaryTierFactory
|
||||
from the registered tier type.
|
||||
"""
|
||||
self._vllm_config = vllm_config
|
||||
self._offloading_spec = offloading_spec
|
||||
self._primary_kv_view: memoryview = primary_kv_view
|
||||
self.tier_type = tier_type
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ from vllm.v1.kv_offload.tiering.base import (
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.kv_offload.base import OffloadingSpec
|
||||
|
||||
|
||||
class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
@@ -37,7 +37,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
offloading_spec: "OffloadingSpec",
|
||||
primary_kv_view: memoryview,
|
||||
tier_type: str,
|
||||
custom_param: int = 0,
|
||||
@@ -49,7 +49,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
custom_param: Dummy parameter demonstrating custom args.
|
||||
"""
|
||||
super().__init__(
|
||||
vllm_config=vllm_config,
|
||||
offloading_spec=offloading_spec,
|
||||
primary_kv_view=primary_kv_view,
|
||||
tier_type=tier_type,
|
||||
)
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import TYPE_CHECKING
|
||||
from vllm.v1.kv_offload.tiering.base import SecondaryTierManager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.kv_offload.base import OffloadingSpec
|
||||
|
||||
|
||||
class SecondaryTierFactory:
|
||||
@@ -29,7 +29,7 @@ class SecondaryTierFactory:
|
||||
cls,
|
||||
tier_config: dict,
|
||||
primary_kv_view: memoryview,
|
||||
vllm_config: "VllmConfig",
|
||||
offloading_spec: "OffloadingSpec",
|
||||
) -> SecondaryTierManager:
|
||||
config = tier_config.copy()
|
||||
|
||||
@@ -45,7 +45,7 @@ class SecondaryTierFactory:
|
||||
|
||||
tier_cls = cls._registry[tier_type]()
|
||||
return tier_cls(
|
||||
vllm_config=vllm_config,
|
||||
offloading_spec=offloading_spec,
|
||||
primary_kv_view=primary_kv_view,
|
||||
tier_type=tier_type,
|
||||
**config,
|
||||
|
||||
@@ -124,7 +124,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
for i, tier_config in enumerate(self.secondary_tier_configs):
|
||||
try:
|
||||
tier = SecondaryTierFactory.create_secondary_tier(
|
||||
tier_config, primary_kv_view, self.vllm_config
|
||||
tier_config, primary_kv_view, self
|
||||
)
|
||||
secondary_tiers.append(tier)
|
||||
logger.info(
|
||||
|
||||
Reference in New Issue
Block a user