Support DeepSeek-V4 AMD Quark NVFP4 with emulation kernel (#47972)
Signed-off-by: Jimmy Adams <Adam_ji@outlook.com> Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: huji <huidong.ji@amd.com> Signed-off-by: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: huji <huidong.ji@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Andreas Karatzas
huji
Cursor
parent
0de0362ea1
commit
da788334bc
@@ -3341,6 +3341,29 @@ steps:
|
||||
- export VLLM_USE_DEEP_GEMM=0
|
||||
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-fp8.txt --tp-size=4
|
||||
|
||||
- label: LM Eval Large Models (8xB200-8xMI355) # TBD
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||
dind: false
|
||||
agent_pool: mi355_8
|
||||
optional: true
|
||||
num_gpus: 8
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/models/
|
||||
- vllm/model_executor/model_loader/
|
||||
- vllm/model_executor/layers/quantization/
|
||||
- vllm/v1/attention/backends/
|
||||
- vllm/v1/attention/selector.py
|
||||
- vllm/model_executor/layers/layernorm.py
|
||||
- csrc/
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
- tests/evals/
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-gfx950-large.txt
|
||||
|
||||
#--------------------------------------------------------- mi355 · examples ----------------------------------------------------------#
|
||||
|
||||
- label: Examples # TBD
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
model_name: "amd/DeepSeek-V4-Flash-NVFP4"
|
||||
accuracy_threshold: 0.92
|
||||
num_questions: 1319
|
||||
num_fewshot: 8
|
||||
startup_max_wait_seconds: 1800
|
||||
server_args: >-
|
||||
--tensor-parallel-size 8
|
||||
--max-model-len 4096
|
||||
--trust-remote-code
|
||||
--kv-cache-dtype fp8
|
||||
--tokenizer-mode deepseek_v4
|
||||
--compilation-config '{"mode": 3, "cudagraph_mode": "FULL_DECODE_ONLY"}'
|
||||
env:
|
||||
VLLM_ROCM_USE_AITER: "1"
|
||||
VLLM_ROCM_USE_AITER_MOE: "1"
|
||||
@@ -0,0 +1,15 @@
|
||||
model_name: "amd/DeepSeek-V4-Pro-NVFP4"
|
||||
accuracy_threshold: 0.92
|
||||
num_questions: 1319
|
||||
num_fewshot: 8
|
||||
startup_max_wait_seconds: 3600
|
||||
server_args: >-
|
||||
--tensor-parallel-size 8
|
||||
--max-model-len 4096
|
||||
--trust-remote-code
|
||||
--kv-cache-dtype fp8
|
||||
--tokenizer-mode deepseek_v4
|
||||
--compilation-config '{"mode": 3, "cudagraph_mode": "FULL_DECODE_ONLY"}'
|
||||
env:
|
||||
VLLM_ROCM_USE_AITER: "1"
|
||||
VLLM_ROCM_USE_AITER_MOE: "1"
|
||||
@@ -0,0 +1,8 @@
|
||||
# GFX950 model configurations for GSM8K evaluation
|
||||
# TP8 models only
|
||||
# DeepSeek-V4-Pro-NVFP4.yaml is temporarily disabled: repeated CI failures due to
|
||||
# cold NFS reads of the 850GiB checkpoint exceeding startup_max_wait_seconds
|
||||
# (see https://github.com/vllm-project/vllm/pull/47972). Re-enable once the
|
||||
# startup timeout fix has landed and is verified green.
|
||||
# DeepSeek-V4-Pro-NVFP4.yaml
|
||||
DeepSeek-V4-Flash-NVFP4.yaml
|
||||
@@ -19,8 +19,10 @@ from packaging import version
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501
|
||||
QuarkConfig,
|
||||
QuarkLinearMethod,
|
||||
QuarkW8A8Fp8,
|
||||
QuarkW8A8Fp8PerBlock,
|
||||
QuarkW8A8Int8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501
|
||||
@@ -80,6 +82,101 @@ def enable_pickle(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
|
||||
def test_quark_config_has_no_model_specific_fused_mappings():
|
||||
config = QuarkConfig({})
|
||||
|
||||
assert "gate_up_proj" not in config.packed_modules_mapping
|
||||
assert "fused_wqa_wkv" not in config.packed_modules_mapping
|
||||
|
||||
|
||||
def test_quark_config_preserves_existing_packed_modules_mapping():
|
||||
class CustomQuarkConfig(QuarkConfig):
|
||||
packed_modules_mapping = {"custom_proj": ["a", "b"]}
|
||||
|
||||
config = CustomQuarkConfig({})
|
||||
|
||||
assert config.packed_modules_mapping["custom_proj"] == ["a", "b"]
|
||||
|
||||
|
||||
def test_quark_fp8_w8a8_detects_per_block_config():
|
||||
config = QuarkConfig({})
|
||||
weight_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
}
|
||||
input_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
}
|
||||
|
||||
assert config._is_fp8_w8a8(weight_config, input_config)
|
||||
|
||||
|
||||
def test_quark_fp8_w8a8_rejects_per_block_static_input():
|
||||
config = QuarkConfig({})
|
||||
weight_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
}
|
||||
input_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": False,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
}
|
||||
|
||||
assert not config._is_fp8_w8a8(weight_config, input_config)
|
||||
|
||||
|
||||
def test_quark_fp8_w8a8_rejects_per_block_group_size_mismatch():
|
||||
config = QuarkConfig({})
|
||||
weight_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"block_size": [128, 128],
|
||||
"symmetric": True,
|
||||
}
|
||||
input_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 64,
|
||||
"symmetric": True,
|
||||
}
|
||||
|
||||
assert not config._is_fp8_w8a8(weight_config, input_config)
|
||||
|
||||
|
||||
def test_quark_w8a8_fp8_per_block_requires_block_size():
|
||||
weight_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"is_dynamic": False,
|
||||
"symmetric": True,
|
||||
}
|
||||
input_config = {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"is_dynamic": True,
|
||||
"group_size": 128,
|
||||
"symmetric": True,
|
||||
}
|
||||
|
||||
with pytest.raises(ValueError, match="requires `block_size`"):
|
||||
QuarkW8A8Fp8PerBlock(weight_config, input_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.parametrize("tp", [1])
|
||||
def test_quark_fp8_w_per_tensor_a_per_tensor(vllm_runner, kv_cache_dtype, tp):
|
||||
|
||||
@@ -193,6 +193,7 @@ def select_nvfp4_moe_backend(
|
||||
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
|
||||
NvFp4MoeBackend.VLLM_CUTLASS,
|
||||
NvFp4MoeBackend.MARLIN,
|
||||
NvFp4MoeBackend.EMULATION,
|
||||
NvFp4MoeBackend.HUMMING,
|
||||
}
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ from vllm.model_executor.layers.quantization.quark.schemes import (
|
||||
QuarkScheme,
|
||||
QuarkW4A8_MXFP4_FP8,
|
||||
QuarkW8A8Fp8,
|
||||
QuarkW8A8Fp8PerBlock,
|
||||
QuarkW8A8Int8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
@@ -66,6 +67,14 @@ class QuarkConfig(QuantizationConfig):
|
||||
if kv_cache_group is None:
|
||||
kv_cache_group = []
|
||||
self.quant_config = quant_config
|
||||
# Copy the class-level default (which a subclass may override, e.g. the
|
||||
# DeepSeek-V4 Quark config) so per-instance edits don't mutate the class.
|
||||
# Read from the class rather than ``self`` because the base
|
||||
# ``QuantizationConfig.__init__`` sets an empty instance-level
|
||||
# ``packed_modules_mapping`` that would otherwise shadow the override.
|
||||
self.packed_modules_mapping = dict(
|
||||
getattr(type(self), "packed_modules_mapping", {})
|
||||
)
|
||||
self.kv_cache_group = kv_cache_group
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.pack_method = pack_method
|
||||
@@ -341,11 +350,30 @@ class QuarkConfig(QuantizationConfig):
|
||||
"per_tensor",
|
||||
"per_channel",
|
||||
]
|
||||
block_size = list(weight_quant.get("block_size") or [])
|
||||
is_per_block_weight = (
|
||||
weight_quant.get("qscheme") == "per_block"
|
||||
and block_size == [128, 128]
|
||||
and weight_quant.get("symmetric") is True
|
||||
)
|
||||
|
||||
if not (is_fp8_dtype and is_static_weight and is_per_tensor_or_channel_weight):
|
||||
if not is_fp8_dtype or not is_static_weight:
|
||||
return False
|
||||
|
||||
# Dynamic quantization is always supported if weights supported.
|
||||
if is_per_block_weight:
|
||||
if not input_quant.get("is_dynamic"):
|
||||
return False
|
||||
return (
|
||||
input_quant.get("qscheme") == "per_group"
|
||||
and input_quant.get("group_size") == block_size[1]
|
||||
and input_quant.get("symmetric") is True
|
||||
)
|
||||
|
||||
if not is_per_tensor_or_channel_weight:
|
||||
return False
|
||||
|
||||
# Dynamic quantization is always supported if tensor/channel weights
|
||||
# are supported.
|
||||
if input_quant.get("is_dynamic"):
|
||||
return True
|
||||
|
||||
@@ -640,6 +668,8 @@ class QuarkConfig(QuantizationConfig):
|
||||
QuarkW8A8Fp8.get_min_capability(), error=False
|
||||
)
|
||||
if is_fp8_w8a8_supported:
|
||||
if weight_config.get("qscheme") == "per_block":
|
||||
return QuarkW8A8Fp8PerBlock(weight_config, input_config)
|
||||
return QuarkW8A8Fp8(weight_config, input_config)
|
||||
elif self._is_static_tensor_w8a8(weight_config, input_config):
|
||||
weight_qscheme = cast(str, weight_config.get("qscheme"))
|
||||
|
||||
@@ -1590,15 +1590,16 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
Convert NVFP4 MoE weights into kernel format and setup the kernel.
|
||||
"""
|
||||
|
||||
if not torch.allclose(
|
||||
# Match existing NVFP4 MoE paths: fused w13 uses the w1 global scale.
|
||||
if self.moe.is_act_and_mul and not torch.allclose(
|
||||
layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1]
|
||||
):
|
||||
raise ValueError("Different global scales for w1 and w3 is not supported.")
|
||||
logger.warning_once(
|
||||
"w1_weight_scale_2 must match w3_weight_scale_2. "
|
||||
"Accuracy may be affected."
|
||||
)
|
||||
|
||||
# Use a single gscale for w13
|
||||
w13_weight_scale_2 = torch.maximum(
|
||||
layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1]
|
||||
).contiguous()
|
||||
w13_weight_scale_2 = layer.w13_weight_scale_2[:, 0].contiguous()
|
||||
|
||||
w2_weight_scale_2 = layer.w2_weight_scale_2
|
||||
|
||||
|
||||
@@ -5,12 +5,13 @@ from .quark_nvfp4 import QuarkNVFP4
|
||||
from .quark_ocp_mx import QuarkOCP_MX
|
||||
from .quark_scheme import QuarkScheme
|
||||
from .quark_w4a8_mxfp4_fp8 import QuarkW4A8_MXFP4_FP8
|
||||
from .quark_w8a8_fp8 import QuarkW8A8Fp8
|
||||
from .quark_w8a8_fp8 import QuarkW8A8Fp8, QuarkW8A8Fp8PerBlock
|
||||
from .quark_w8a8_int8 import QuarkW8A8Int8
|
||||
|
||||
__all__ = [
|
||||
"QuarkScheme",
|
||||
"QuarkW8A8Fp8",
|
||||
"QuarkW8A8Fp8PerBlock",
|
||||
"QuarkW8A8Int8",
|
||||
"QuarkOCP_MX",
|
||||
"QuarkW4A8_MXFP4_FP8",
|
||||
|
||||
@@ -9,13 +9,18 @@ from torch.nn import Parameter
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
init_fp8_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
|
||||
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
create_fp8_scale_parameter,
|
||||
create_fp8_weight_parameter,
|
||||
validate_fp8_block_shape,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
@@ -24,13 +29,14 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
requantize_with_max_scale,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
BlockQuantScaleParameter,
|
||||
ChannelQuantScaleParameter,
|
||||
ModelWeightParameter,
|
||||
PerTensorScaleParameter,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
__all__ = ["QuarkW8A8Fp8"]
|
||||
__all__ = ["QuarkW8A8Fp8", "QuarkW8A8Fp8PerBlock"]
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -46,6 +52,11 @@ class QuarkW8A8Fp8(QuarkScheme):
|
||||
self.is_static_input_scheme = not cast(bool, input_config.get("is_dynamic"))
|
||||
self.input_qscheme = cast(str, input_config.get("qscheme"))
|
||||
|
||||
if self.weight_qscheme == "per_block":
|
||||
raise ValueError(
|
||||
"Quark W8A8 FP8 per-block weight quantization should use "
|
||||
"QuarkW8A8Fp8PerBlock."
|
||||
)
|
||||
per_token_activation = (
|
||||
not self.is_static_input_scheme and self.input_qscheme == "per_channel"
|
||||
)
|
||||
@@ -196,3 +207,103 @@ class QuarkW8A8Fp8(QuarkScheme):
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.fp8_linear.apply_weights(layer, x, bias)
|
||||
|
||||
|
||||
class QuarkW8A8Fp8PerBlock(QuarkScheme):
|
||||
def __init__(
|
||||
self, weight_config: dict[str, Any], input_config: dict[str, Any] | None
|
||||
):
|
||||
self.weight_qscheme = cast(str, weight_config.get("qscheme"))
|
||||
if self.weight_qscheme != "per_block":
|
||||
raise ValueError("QuarkW8A8Fp8PerBlock requires per-block weights.")
|
||||
block_size = weight_config.get("block_size")
|
||||
if not block_size:
|
||||
raise ValueError(
|
||||
"Quark W8A8 FP8 per-block weight quantization requires "
|
||||
"`block_size` in the weight quantization config."
|
||||
)
|
||||
self.weight_config = weight_config
|
||||
self.weight_block_size = list(block_size)
|
||||
self.activation_quant_key = kFp8Dynamic128Sym
|
||||
self.weight_quant_key = kFp8Static128BlockSym
|
||||
self.out_dtype = torch.get_default_dtype()
|
||||
self.input_dtype = get_current_vllm_config().model_config.dtype
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return QuarkW8A8Fp8.get_min_capability()
|
||||
|
||||
def process_weights_after_loading(self, layer) -> None:
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
output_partition_sizes: list[int],
|
||||
input_size_per_partition: int,
|
||||
params_dtype: torch.dtype,
|
||||
weight_loader: Callable,
|
||||
**kwargs,
|
||||
):
|
||||
input_size = kwargs.get("input_size", input_size_per_partition)
|
||||
output_size = kwargs.get("output_size", sum(output_partition_sizes))
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
|
||||
layer.logical_widths = output_partition_sizes
|
||||
layer.input_size_per_partition = input_size_per_partition
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.orig_dtype = params_dtype
|
||||
layer.weight_block_size = self.weight_block_size
|
||||
|
||||
validate_fp8_block_shape(
|
||||
layer,
|
||||
input_size,
|
||||
output_size,
|
||||
input_size_per_partition,
|
||||
output_partition_sizes,
|
||||
self.weight_block_size,
|
||||
)
|
||||
|
||||
weight = create_fp8_weight_parameter(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition,
|
||||
weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
scale_dtype = (
|
||||
torch.float8_e8m0fnu
|
||||
if self.weight_config.get("scale_type") == "float8_e8m0fnu"
|
||||
else None
|
||||
)
|
||||
weight_scale = create_fp8_scale_parameter(
|
||||
BlockQuantScaleParameter,
|
||||
output_partition_sizes,
|
||||
input_size_per_partition,
|
||||
self.weight_block_size,
|
||||
weight_loader,
|
||||
scale_dtype=scale_dtype,
|
||||
)
|
||||
# DeepSeek V4 weight mappers route checkpoint ".scale" tensors here.
|
||||
layer.register_parameter("weight_scale_inv", weight_scale)
|
||||
|
||||
self.fp8_linear = init_fp8_linear_kernel(
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
weight_shape=layer.weight.shape,
|
||||
input_dtype=self.input_dtype,
|
||||
out_dtype=self.out_dtype,
|
||||
module_name=self.__class__.__name__,
|
||||
)
|
||||
logger.info_once(
|
||||
"Selected %s for QuarkW8A8Fp8PerBlock",
|
||||
type(self.fp8_linear).__name__,
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.fp8_linear.apply_weights(layer, x, bias)
|
||||
|
||||
@@ -750,6 +750,12 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
def _resolve_param_name(name: str) -> str:
|
||||
inv_name = f"{name}_inv"
|
||||
if name not in params_dict and inv_name in params_dict:
|
||||
return inv_name
|
||||
return name
|
||||
|
||||
# TP for attention
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
@@ -803,10 +809,11 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
break
|
||||
param = params_dict[name]
|
||||
param_name = _resolve_param_name(name)
|
||||
param = params_dict[param_name]
|
||||
weight_loader = param.weight_loader
|
||||
weight_loader(param, loaded_weight, shard_id)
|
||||
loaded_params.add(name)
|
||||
loaded_params.add(param_name)
|
||||
break
|
||||
else:
|
||||
if ".experts." in name:
|
||||
@@ -857,12 +864,13 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
else:
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
param = params_dict[name]
|
||||
param_name = _resolve_param_name(name)
|
||||
param = params_dict[param_name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
loaded_params.add(name)
|
||||
loaded_params.add(param_name)
|
||||
continue
|
||||
|
||||
return loaded_params
|
||||
@@ -934,6 +942,7 @@ def _make_deepseek_v4_weights_mapper(
|
||||
"head.weight": "lm_head.weight",
|
||||
"embed.weight": "embed_tokens.weight",
|
||||
".ffn.gate.bias": ".ffn.gate.e_score_correction_bias",
|
||||
".input_scale": ".input_scale_2",
|
||||
},
|
||||
orig_to_new_substr=substr_map,
|
||||
)
|
||||
@@ -945,6 +954,10 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
|
||||
# Default mapper assumes the original FP4-expert checkpoint layout.
|
||||
# Overridden per-instance in __init__ when expert_dtype != "fp4".
|
||||
hf_to_vllm_mapper = _make_deepseek_v4_weights_mapper("fp4")
|
||||
packed_modules_mapping = {
|
||||
"gate_up_proj": ["w1", "w3"],
|
||||
"fused_wqa_wkv": ["wq_a", "wkv"],
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
Reference in New Issue
Block a user