[Bugfix][CPU] Fix several bugs (#54042)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
+9
-1
@@ -934,6 +934,14 @@ def hf_runner():
|
||||
return HfRunner
|
||||
|
||||
|
||||
def _default_block_size() -> int:
|
||||
if torch.xpu.is_available():
|
||||
return 64
|
||||
if current_platform.is_cpu():
|
||||
return 128
|
||||
return 16
|
||||
|
||||
|
||||
class VllmRunner:
|
||||
"""
|
||||
The default value of some arguments have been modified from
|
||||
@@ -962,7 +970,7 @@ class VllmRunner:
|
||||
dtype: str = "auto",
|
||||
disable_log_stats: bool = True,
|
||||
tensor_parallel_size: int = 1,
|
||||
block_size: int = 16 if not torch.xpu.is_available() else 64,
|
||||
block_size: int = _default_block_size(),
|
||||
enable_chunked_prefill: bool | None = False,
|
||||
enforce_eager: bool | None = False,
|
||||
# Set this to avoid hanging issue
|
||||
|
||||
Reference in New Issue
Block a user