[ROCm][CI] Avoid HIP init at config time via lazy aiter import in Quark OCP-MX (#48015)
Signed-off-by: Dino Music <Dino.Music@amd.com>
This commit is contained in:
@@ -9,7 +9,7 @@ from typing import Any
|
|||||||
import torch
|
import torch
|
||||||
import torch.nn.functional as F
|
import torch.nn.functional as F
|
||||||
|
|
||||||
from vllm._aiter_ops import rocm_aiter_ops
|
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||||
dequant_mxfp4,
|
dequant_mxfp4,
|
||||||
@@ -36,19 +36,15 @@ from .quark_scheme import QuarkScheme
|
|||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
try:
|
# NOTE: Do not import aiter at module scope. Importing aiter eagerly initializes HIP
|
||||||
from aiter.ops.shuffle import shuffle_weight
|
# which can force the engine core to spawn instead of fork.
|
||||||
from aiter.ops.triton.gemm_afp4wfp4 import (
|
# is_aiter_found_and_supported() checks platform + arch + library availability via
|
||||||
gemm_afp4wfp4,
|
# find_spec/amdsmi, so it stays HIP-free.
|
||||||
gemm_afp4wfp4_preshuffled_weight_scales,
|
# Actual aiter imports are deferred to the functions/methods that need them,
|
||||||
)
|
# where HIP initialization is expected.
|
||||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
if is_aiter_found_and_supported():
|
||||||
|
|
||||||
from vllm.utils.torch_utils import direct_register_custom_op
|
from vllm.utils.torch_utils import direct_register_custom_op
|
||||||
|
|
||||||
if rocm_aiter_ops.is_asm_fp4_gemm_dynamic_quant_enabled():
|
|
||||||
from aiter import gemm_a4w4, per_1x32_f4_quant_hip
|
|
||||||
|
|
||||||
def gemm_with_dynamic_quant(
|
def gemm_with_dynamic_quant(
|
||||||
x: torch.Tensor,
|
x: torch.Tensor,
|
||||||
weight: torch.Tensor,
|
weight: torch.Tensor,
|
||||||
@@ -57,6 +53,15 @@ try:
|
|||||||
out_dtype: torch.dtype | None = torch.bfloat16,
|
out_dtype: torch.dtype | None = torch.bfloat16,
|
||||||
x_scales: torch.Tensor | None = None,
|
x_scales: torch.Tensor | None = None,
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
|
from aiter.ops.triton.gemm_afp4wfp4 import (
|
||||||
|
gemm_afp4wfp4,
|
||||||
|
gemm_afp4wfp4_preshuffled_weight_scales,
|
||||||
|
)
|
||||||
|
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||||
|
|
||||||
|
if rocm_use_aiter_fp4_asm_gemm:
|
||||||
|
from aiter import gemm_a4w4, per_1x32_f4_quant_hip
|
||||||
|
|
||||||
M = x.shape[0]
|
M = x.shape[0]
|
||||||
N = weight.shape[0]
|
N = weight.shape[0]
|
||||||
K = weight.shape[1]
|
K = weight.shape[1]
|
||||||
@@ -137,13 +142,12 @@ try:
|
|||||||
fake_impl=gemm_with_dynamic_quant_fake,
|
fake_impl=gemm_with_dynamic_quant_fake,
|
||||||
dispatch_key=current_platform.dispatch_key,
|
dispatch_key=current_platform.dispatch_key,
|
||||||
)
|
)
|
||||||
except (ImportError, AttributeError, RuntimeError):
|
elif current_platform.is_rocm():
|
||||||
if current_platform.is_rocm():
|
logger.warning(
|
||||||
logger.warning(
|
"AITER is not found or not supported on the current platform, "
|
||||||
"AITER is not found or QuarkOCP_MX is not supported on the current "
|
"QuarkOCP_MX will fall back to emulation."
|
||||||
"platform. QuarkOCP_MX quantization will not be available."
|
"Native MXFP4/MXFP6 acceleration will not be available."
|
||||||
)
|
)
|
||||||
dynamic_mxfp4_quant = gemm_afp4wfp4 = None
|
|
||||||
|
|
||||||
|
|
||||||
class QuarkOCP_MX(QuarkScheme):
|
class QuarkOCP_MX(QuarkScheme):
|
||||||
@@ -211,8 +215,8 @@ class QuarkOCP_MX(QuarkScheme):
|
|||||||
rocm_aiter_ops.is_asm_fp4_gemm_dynamic_quant_enabled()
|
rocm_aiter_ops.is_asm_fp4_gemm_dynamic_quant_enabled()
|
||||||
)
|
)
|
||||||
|
|
||||||
if not self.emulate and (dynamic_mxfp4_quant is None or gemm_afp4wfp4 is None):
|
if not self.emulate and not is_aiter_found_and_supported():
|
||||||
# Currently need these kernels if not emulating
|
# Currently need AITER kernels if not emulating
|
||||||
raise NotImplementedError(
|
raise NotImplementedError(
|
||||||
f"{self.__class__.__name__} requires AITER to be installed "
|
f"{self.__class__.__name__} requires AITER to be installed "
|
||||||
"for non-emulation mode! Please refer to "
|
"for non-emulation mode! Please refer to "
|
||||||
@@ -261,6 +265,8 @@ class QuarkOCP_MX(QuarkScheme):
|
|||||||
def process_dynamic_mxfp4_weights_after_loading(
|
def process_dynamic_mxfp4_weights_after_loading(
|
||||||
self, layer: torch.nn.Module
|
self, layer: torch.nn.Module
|
||||||
) -> None:
|
) -> None:
|
||||||
|
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||||
|
|
||||||
w_q, w_s = dynamic_mxfp4_quant(layer.weight)
|
w_q, w_s = dynamic_mxfp4_quant(layer.weight)
|
||||||
layer.weight_scale = torch.nn.Parameter(w_s.T.contiguous(), requires_grad=False)
|
layer.weight_scale = torch.nn.Parameter(w_s.T.contiguous(), requires_grad=False)
|
||||||
layer.weight = torch.nn.Parameter(w_q, requires_grad=False)
|
layer.weight = torch.nn.Parameter(w_q, requires_grad=False)
|
||||||
@@ -279,6 +285,8 @@ class QuarkOCP_MX(QuarkScheme):
|
|||||||
if self.dynamic_mxfp4_quant:
|
if self.dynamic_mxfp4_quant:
|
||||||
self.process_dynamic_mxfp4_weights_after_loading(layer)
|
self.process_dynamic_mxfp4_weights_after_loading(layer)
|
||||||
elif self.rocm_use_aiter_fp4_asm_gemm:
|
elif self.rocm_use_aiter_fp4_asm_gemm:
|
||||||
|
from aiter.ops.shuffle import shuffle_weight
|
||||||
|
|
||||||
# shuffle weight scale
|
# shuffle weight scale
|
||||||
weight_scale_shuffle = layer.weight_scale.data
|
weight_scale_shuffle = layer.weight_scale.data
|
||||||
sm, sn = weight_scale_shuffle.shape
|
sm, sn = weight_scale_shuffle.shape
|
||||||
|
|||||||
Reference in New Issue
Block a user