[kv_offload+HMA][3/N]: Remove block_size from KVEvents (#36644)
Signed-off-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -424,7 +424,7 @@ class OffloadingConnectorScheduler:
|
||||
parent_block_hash=None,
|
||||
token_ids=[],
|
||||
lora_id=None,
|
||||
block_size=event.block_size,
|
||||
block_size=0,
|
||||
medium=event.medium,
|
||||
lora_name=None,
|
||||
)
|
||||
|
||||
@@ -79,7 +79,6 @@ class PrepareStoreOutput:
|
||||
@dataclass
|
||||
class OffloadingEvent:
|
||||
keys: list[OffloadKey]
|
||||
block_size: int
|
||||
medium: str
|
||||
# True if blocks are removed, False if stored
|
||||
removed: bool
|
||||
|
||||
@@ -33,12 +33,10 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
block_size: int,
|
||||
num_blocks: int,
|
||||
cache_policy: Literal["lru", "arc"] = "lru",
|
||||
enable_events: bool = False,
|
||||
):
|
||||
self.block_size: int = block_size
|
||||
self.medium: str = CPULoadStoreSpec.medium()
|
||||
self._num_blocks: int = num_blocks
|
||||
self._num_allocated_blocks: int = 0
|
||||
@@ -145,7 +143,6 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
self.events.append(
|
||||
OffloadingEvent(
|
||||
keys=to_evict,
|
||||
block_size=self.block_size,
|
||||
medium=self.medium,
|
||||
removed=True,
|
||||
)
|
||||
@@ -188,7 +185,6 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
self.events.append(
|
||||
OffloadingEvent(
|
||||
keys=stored_keys,
|
||||
block_size=self.block_size,
|
||||
medium=self.medium,
|
||||
removed=False,
|
||||
)
|
||||
|
||||
@@ -60,12 +60,7 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
kv_events_config is not None and kv_events_config.enable_kv_cache_events
|
||||
)
|
||||
|
||||
assert len(self.gpu_block_size) == 1
|
||||
gpu_block_size = self.gpu_block_size[0]
|
||||
offloaded_block_size = gpu_block_size * self.block_size_factor
|
||||
|
||||
self._manager = CPUOffloadingManager(
|
||||
block_size=offloaded_block_size,
|
||||
num_blocks=self.num_blocks,
|
||||
cache_policy=self.eviction_policy, # type: ignore[arg-type]
|
||||
enable_events=enable_events,
|
||||
|
||||
Reference in New Issue
Block a user