[Hardware][NIXL] set default kv buffer type for different platform (#36438)
Signed-off-by: zhenwei-intel <zhenwei.liu@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
co-authored by
Kunshang Ji
parent
76c6e6da08
commit
f22d6e0267
@@ -998,7 +998,10 @@ class NixlConnectorWorker:
|
||||
|
||||
# KV Caches and nixl tracking data.
|
||||
self.device_type = current_platform.device_type
|
||||
self.kv_buffer_device: str = vllm_config.kv_transfer_config.kv_buffer_device
|
||||
kv_buffer_device = vllm_config.kv_transfer_config.kv_buffer_device
|
||||
if kv_buffer_device is None:
|
||||
raise ValueError("kv_buffer_device must be set for NixlConnector")
|
||||
self.kv_buffer_device: str = kv_buffer_device
|
||||
if self.device_type not in _NIXL_SUPPORTED_DEVICE:
|
||||
raise RuntimeError(f"{self.device_type} is not supported.")
|
||||
elif self.kv_buffer_device not in _NIXL_SUPPORTED_DEVICE[self.device_type]:
|
||||
|
||||
Reference in New Issue
Block a user