[Platform] Add current_platform.num_compute_units interface (#35042)
Signed-off-by: Kunshang Ji <kunshang.ji@intel.com> Signed-off-by: Kunshang Ji <jikunshang95@gmail.com>
This commit is contained in:
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.quantization.utils.int8_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import ScalarType, scalar_types
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
from .quant_utils import pack_cols, unpack_cols
|
||||
|
||||
@@ -271,7 +272,7 @@ def marlin_make_workspace_new(
|
||||
) -> torch.Tensor:
|
||||
# In the new marlin kernel, we use the num of threadblocks as workspace
|
||||
# size. The num of threadblocks is sms_count * max_blocks_per_sm.
|
||||
sms = torch.cuda.get_device_properties(device).multi_processor_count
|
||||
sms = num_compute_units(device.index)
|
||||
return torch.zeros(
|
||||
sms * max_blocks_per_sm, dtype=torch.int, device=device, requires_grad=False
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user