[misc][cuda] use nvml to avoid accidentally cuda initialization (#6007)
This commit is contained in:
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
|
||||
CompressionFormat, QuantizationArgs, QuantizationStrategy,
|
||||
find_first_name_or_class_match)
|
||||
from vllm.utils import get_device_capability_stateless
|
||||
|
||||
|
||||
class CompressedTensorsConfig(QuantizationConfig):
|
||||
@@ -84,7 +85,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
||||
return []
|
||||
|
||||
def _check_gptq_and_marlin_can_run(self):
|
||||
capability = torch.cuda.get_device_capability()
|
||||
capability = get_device_capability_stateless()
|
||||
capability = capability[0] * 10 + capability[1]
|
||||
if capability < 80:
|
||||
raise RuntimeError("The quantization config is not supported for ",
|
||||
|
||||
@@ -10,7 +10,7 @@ from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig, QuantizeMethodBase)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.utils import print_warning_once
|
||||
from vllm.utils import get_device_capability_stateless, print_warning_once
|
||||
|
||||
ACTIVATION_SCHEMES = ["static", "dynamic"]
|
||||
|
||||
@@ -18,7 +18,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
def cutlass_fp8_supported() -> bool:
|
||||
capability = torch.cuda.get_device_capability()
|
||||
capability = get_device_capability_stateless()
|
||||
capability = capability[0] * 10 + capability[1]
|
||||
|
||||
return ops.cutlass_scaled_mm_supports_fp8(capability)
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
|
||||
set_weight_attrs)
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig)
|
||||
from vllm.utils import get_device_capability_stateless
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -165,7 +166,7 @@ class GPTQMarlinConfig(QuantizationConfig):
|
||||
return False
|
||||
|
||||
# If the capability of the device is too low, cannot convert.
|
||||
major, minor = torch.cuda.get_device_capability()
|
||||
major, minor = get_device_capability_stateless()
|
||||
device_capability = major * 10 + minor
|
||||
if device_capability < cls.get_min_capability():
|
||||
return False
|
||||
|
||||
@@ -12,8 +12,9 @@ from vllm.model_executor.layers.quantization.utils.marlin_perms import (
|
||||
marlin_perm, marlin_scale_perm, marlin_scale_perm_single)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_pack_factor, quantize_weights, sort_weights)
|
||||
from vllm.utils import get_device_capability_stateless
|
||||
|
||||
__cuda_arch = torch.cuda.get_device_capability()
|
||||
__cuda_arch = get_device_capability_stateless()
|
||||
|
||||
MARLIN_TILE = 16
|
||||
|
||||
|
||||
Reference in New Issue
Block a user