[hardware][misc] introduce platform abstraction (#6080)

This commit is contained in:
youkaichao
2024-07-02 20:12:22 -07:00
committed by GitHub
parent 9d6a8daa87
commit 482045ee77
16 changed files with 113 additions and 29 deletions
@@ -14,7 +14,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
CompressionFormat, QuantizationArgs, QuantizationStrategy,
find_first_name_or_class_match)
from vllm.utils import get_device_capability_stateless
from vllm.platforms import current_platform
class CompressedTensorsConfig(QuantizationConfig):
@@ -85,7 +85,7 @@ class CompressedTensorsConfig(QuantizationConfig):
return []
def _check_gptq_and_marlin_can_run(self):
capability = get_device_capability_stateless()
capability = current_platform.get_device_capability()
capability = capability[0] * 10 + capability[1]
if capability < 80:
raise RuntimeError("The quantization config is not supported for ",
@@ -12,7 +12,8 @@ from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig, QuantizeMethodBase)
from vllm.model_executor.utils import set_weight_attrs
from vllm.utils import get_device_capability_stateless, print_warning_once
from vllm.platforms import current_platform
from vllm.utils import print_warning_once
ACTIVATION_SCHEMES = ["static", "dynamic"]
@@ -20,7 +21,7 @@ logger = init_logger(__name__)
def cutlass_fp8_supported() -> bool:
capability = get_device_capability_stateless()
capability = current_platform.get_device_capability()
capability = capability[0] * 10 + capability[1]
return ops.cutlass_scaled_mm_supports_fp8(capability)
@@ -12,7 +12,7 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig)
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.utils import get_device_capability_stateless
from vllm.platforms import current_platform
logger = init_logger(__name__)
@@ -173,7 +173,7 @@ class GPTQMarlinConfig(QuantizationConfig):
return False
# If the capability of the device is too low, cannot convert.
major, minor = get_device_capability_stateless()
major, minor = current_platform.get_device_capability()
device_capability = major * 10 + minor
if device_capability < cls.get_min_capability():
return False
@@ -12,9 +12,9 @@ from vllm.model_executor.layers.quantization.utils.marlin_perms import (
marlin_perm, marlin_scale_perm, marlin_scale_perm_single)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_pack_factor, quantize_weights, sort_weights)
from vllm.utils import get_device_capability_stateless
from vllm.platforms import current_platform
__cuda_arch = get_device_capability_stateless()
__cuda_arch = current_platform.get_device_capability()
MARLIN_TILE = 16