[Nixl][P/D] Add cuda2cpu support (HD->DH transfer) (#24690)
Signed-off-by: Chenxi Yang <cxyang@fb.com> Co-authored-by: Chenxi Yang <cxyang@fb.com>
This commit is contained in:
co-authored by
Chenxi Yang
parent
43227236ec
commit
d0d138bc55
@@ -4059,10 +4059,9 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
self.drafter.validate_same_kv_cache_group(kv_cache_config)
|
||||
|
||||
if has_kv_transfer_group():
|
||||
get_kv_transfer_group().register_kv_caches(kv_caches)
|
||||
if self.device.type == 'xpu':
|
||||
get_kv_transfer_group().set_host_xfer_buffer_ops(
|
||||
copy_kv_blocks)
|
||||
kv_transfer_group = get_kv_transfer_group()
|
||||
kv_transfer_group.register_kv_caches(kv_caches)
|
||||
kv_transfer_group.set_host_xfer_buffer_ops(copy_kv_blocks)
|
||||
|
||||
if self.dcp_world_size > 1:
|
||||
layer_names = self.attn_groups[0][0].layer_names
|
||||
|
||||
Reference in New Issue
Block a user