[Bugfix][KV Connector] Fix Mooncake physical-block transfer length (#54272)
Signed-off-by: zhewenl <zhewenl@users.noreply.github.com> Co-authored-by: zhewenl <zhewenl@users.noreply.github.com>
This commit is contained in:
@@ -1675,11 +1675,24 @@ class MooncakeConnectorWorker:
|
||||
block_len = region_cache.stride(0) * region_cache.element_size()
|
||||
region_base_addresses.append(base_addr)
|
||||
|
||||
kv_block_len = (
|
||||
layer_spec.page_size_bytes
|
||||
if isinstance(layer_spec, AttentionSpec) and block_is_contiguous
|
||||
else block_len
|
||||
)
|
||||
if isinstance(layer_spec, AttentionSpec) and block_is_contiguous:
|
||||
assert (
|
||||
layer_spec.page_size_bytes
|
||||
% self._physical_blocks_per_logical_kv_block
|
||||
== 0
|
||||
)
|
||||
kv_block_len = (
|
||||
layer_spec.page_size_bytes
|
||||
// self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
else:
|
||||
kv_block_len = block_len
|
||||
if kv_block_len > block_len:
|
||||
raise RuntimeError(
|
||||
"Mooncake transfer length exceeds physical block stride "
|
||||
f"for {layer_name}: kv_block_len={kv_block_len}, "
|
||||
f"block_len={block_len}."
|
||||
)
|
||||
self.block_len_per_layer.append(block_len)
|
||||
self.kv_block_len_per_layer.append(kv_block_len)
|
||||
self.registered_layer_names.append(layer_name)
|
||||
|
||||
Reference in New Issue
Block a user