[Bugfix] Fix offloading set_ overflow for packed non-uniform KV caches (#48530)
Signed-off-by: Elvir Crncevic <elvircrn@gmail.com>
This commit is contained in:
@@ -92,16 +92,17 @@ class OffloadingConnectorWorker:
|
|||||||
if layer_is_packed[layer_name]
|
if layer_is_packed[layer_name]
|
||||||
else page
|
else page
|
||||||
)
|
)
|
||||||
|
raw = torch.empty(
|
||||||
|
0,
|
||||||
|
dtype=torch.int8,
|
||||||
|
device=layer_kv_cache.device,
|
||||||
|
).set_(layer_kv_cache.untyped_storage())
|
||||||
tensors_per_block[layer_name] = (
|
tensors_per_block[layer_name] = (
|
||||||
torch.tensor(
|
torch.as_strided(
|
||||||
[],
|
raw,
|
||||||
dtype=torch.int8,
|
|
||||||
device=layer_kv_cache.device,
|
|
||||||
).set_(
|
|
||||||
layer_kv_cache.untyped_storage(),
|
|
||||||
byte_offset,
|
|
||||||
(num_blocks, page),
|
(num_blocks, page),
|
||||||
(block_stride_bytes, 1),
|
(block_stride_bytes, 1),
|
||||||
|
byte_offset,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
page_size_bytes[layer_name] = layer_kv_cache_spec.page_size_bytes
|
page_size_bytes[layer_name] = layer_kv_cache_spec.page_size_bytes
|
||||||
|
|||||||
Reference in New Issue
Block a user