[Bugfix][CPU] Fix several bugs (#54042)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
+9
-1
@@ -934,6 +934,14 @@ def hf_runner():
|
||||
return HfRunner
|
||||
|
||||
|
||||
def _default_block_size() -> int:
|
||||
if torch.xpu.is_available():
|
||||
return 64
|
||||
if current_platform.is_cpu():
|
||||
return 128
|
||||
return 16
|
||||
|
||||
|
||||
class VllmRunner:
|
||||
"""
|
||||
The default value of some arguments have been modified from
|
||||
@@ -962,7 +970,7 @@ class VllmRunner:
|
||||
dtype: str = "auto",
|
||||
disable_log_stats: bool = True,
|
||||
tensor_parallel_size: int = 1,
|
||||
block_size: int = 16 if not torch.xpu.is_available() else 64,
|
||||
block_size: int = _default_block_size(),
|
||||
enable_chunked_prefill: bool | None = False,
|
||||
enforce_eager: bool | None = False,
|
||||
# Set this to avoid hanging issue
|
||||
|
||||
@@ -44,9 +44,10 @@ def _build(layer_num_heads: list[int]) -> CPUAttentionMetadataBuilder:
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.model_config.dtype = torch.bfloat16
|
||||
vllm_config.model_config.get_num_attention_heads.return_value = MODEL_WIDE_NUM_HEADS
|
||||
vllm_config.cache_config.block_size = 16
|
||||
vllm_config.cache_config.cache_dtype = "auto"
|
||||
kv_cache_spec = SimpleNamespace(num_kv_heads=NUM_KV_HEADS, head_size=64)
|
||||
kv_cache_spec = SimpleNamespace(
|
||||
num_kv_heads=NUM_KV_HEADS, head_size=64, block_size=16
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
|
||||
@@ -87,8 +87,10 @@ def _make_connector_with_fake_worker(
|
||||
hand_shake_latency=0, cycles_before_done=0, do_handshake=True
|
||||
):
|
||||
"""Create a NixlConnector with FakeNixlConnectorWorker."""
|
||||
vllm_config = create_vllm_config(kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG)
|
||||
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=2)
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, block_size=32
|
||||
)
|
||||
kv_cache_config = make_kv_cache_config(block_size=32, num_blocks=2)
|
||||
connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config,
|
||||
|
||||
Reference in New Issue
Block a user