[Fast Start] Support fp4 (#55465)
Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -9,6 +9,7 @@ from vllm._custom_ops import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
cutlass_fp4_supported,
|
||||
nvfp4_weight_padding_bytes,
|
||||
pad_nvfp4_weight_for_cutlass,
|
||||
slice_nvfp4_output,
|
||||
swizzle_blockscale,
|
||||
@@ -36,11 +37,8 @@ class CutlassNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
swizzle_blockscale(layer.weight_scale.data), requires_grad=False
|
||||
)
|
||||
padded_weight, weights_padding_cols = pad_nvfp4_weight_for_cutlass(
|
||||
layer.weight.data
|
||||
)
|
||||
padded_weight, _ = pad_nvfp4_weight_for_cutlass(layer.weight.data)
|
||||
layer.weight = torch.nn.Parameter(padded_weight, requires_grad=False)
|
||||
layer.weights_padding_cols = weights_padding_cols
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -51,7 +49,7 @@ class CutlassNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
output_size = layer.output_size_per_partition
|
||||
output_dtype = x.dtype
|
||||
output_shape = [*x.shape[:-1], output_size]
|
||||
weights_padding_bytes = getattr(layer, "weights_padding_cols", 0)
|
||||
weights_padding_bytes = nvfp4_weight_padding_bytes(layer)
|
||||
|
||||
x_fp4, x_blockscale = scaled_fp4_quant(
|
||||
x,
|
||||
|
||||
@@ -9,6 +9,7 @@ from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
as_quantized_activation,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
nvfp4_weight_padding_bytes,
|
||||
pad_nvfp4_activation_for_cutlass,
|
||||
pad_nvfp4_weight_for_cutlass,
|
||||
slice_nvfp4_output,
|
||||
@@ -130,11 +131,8 @@ class FlashInferCuteDslNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
swizzle_blockscale(layer.weight_scale.data), requires_grad=False
|
||||
)
|
||||
padded_weight, weights_padding_cols = pad_nvfp4_weight_for_cutlass(
|
||||
layer.weight.data
|
||||
)
|
||||
padded_weight, _ = pad_nvfp4_weight_for_cutlass(layer.weight.data)
|
||||
layer.weight = torch.nn.Parameter(padded_weight, requires_grad=False)
|
||||
layer.weights_padding_cols = weights_padding_cols
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -154,7 +152,7 @@ class FlashInferCuteDslNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
)
|
||||
|
||||
x_fp4 = pad_nvfp4_activation_for_cutlass(
|
||||
x_fp4, getattr(layer, "weights_padding_cols", 0)
|
||||
x_fp4, nvfp4_weight_padding_bytes(layer)
|
||||
)
|
||||
|
||||
out = flashinfer_scaled_fp4_mm(
|
||||
@@ -206,11 +204,8 @@ class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
swizzle_blockscale(layer.weight_scale.data), requires_grad=False
|
||||
)
|
||||
padded_weight, weights_padding_cols = pad_nvfp4_weight_for_cutlass(
|
||||
layer.weight.data
|
||||
)
|
||||
padded_weight, _ = pad_nvfp4_weight_for_cutlass(layer.weight.data)
|
||||
layer.weight = torch.nn.Parameter(padded_weight, requires_grad=False)
|
||||
layer.weights_padding_cols = weights_padding_cols
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -219,7 +214,7 @@ class FlashInferCutlassNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
output_size = layer.output_size_per_partition
|
||||
weights_padding_bytes = getattr(layer, "weights_padding_cols", 0)
|
||||
weights_padding_bytes = nvfp4_weight_padding_bytes(layer)
|
||||
|
||||
qa = as_quantized_activation(x, self.input_quant_key())
|
||||
if qa is not None:
|
||||
@@ -343,11 +338,8 @@ class FlashInferCudnnNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
swizzle_blockscale(layer.weight_scale.data), requires_grad=False
|
||||
)
|
||||
padded_weight, weights_padding_cols = pad_nvfp4_weight_for_cutlass(
|
||||
layer.weight.data
|
||||
)
|
||||
padded_weight, _ = pad_nvfp4_weight_for_cutlass(layer.weight.data)
|
||||
layer.weight = torch.nn.Parameter(padded_weight, requires_grad=False)
|
||||
layer.weights_padding_cols = weights_padding_cols
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -358,7 +350,7 @@ class FlashInferCudnnNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
output_size = layer.output_size_per_partition
|
||||
output_dtype = x.dtype
|
||||
output_shape = [*x.shape[:-1], output_size]
|
||||
weights_padding_bytes = getattr(layer, "weights_padding_cols", 0)
|
||||
weights_padding_bytes = nvfp4_weight_padding_bytes(layer)
|
||||
|
||||
x_fp4, x_blockscale = scaled_fp4_quant(
|
||||
x,
|
||||
@@ -408,11 +400,8 @@ class FlashInferB12xNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
swizzle_blockscale(layer.weight_scale.data), requires_grad=False
|
||||
)
|
||||
padded_weight, weights_padding_cols = pad_nvfp4_weight_for_cutlass(
|
||||
layer.weight.data
|
||||
)
|
||||
padded_weight, _ = pad_nvfp4_weight_for_cutlass(layer.weight.data)
|
||||
layer.weight = torch.nn.Parameter(padded_weight, requires_grad=False)
|
||||
layer.weights_padding_cols = weights_padding_cols
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -432,7 +421,7 @@ class FlashInferB12xNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
)
|
||||
|
||||
x_fp4 = pad_nvfp4_activation_for_cutlass(
|
||||
x_fp4, getattr(layer, "weights_padding_cols", 0)
|
||||
x_fp4, nvfp4_weight_padding_bytes(layer)
|
||||
)
|
||||
|
||||
out = flashinfer_scaled_fp4_mm(
|
||||
|
||||
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kNvfp4Dynamic,
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.model_executor.utils import is_weights_pre_processed
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
|
||||
|
||||
@@ -149,8 +150,9 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
return self.quant_config.g1_alphas * self.quant_config.a2_gscale
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.w13_weight_scale_2.data.mul_(layer.w13_input_scale)
|
||||
layer.w2_weight_scale_2.data.mul_(layer.w2_input_scale)
|
||||
if not is_weights_pre_processed():
|
||||
layer.w13_weight_scale_2.data.mul_(layer.w13_input_scale)
|
||||
layer.w2_weight_scale_2.data.mul_(layer.w2_input_scale)
|
||||
# Recompute g1_scale_c since g1_alphas was just fused in-place.
|
||||
# Register as a layer parameter so EPLB rearranges it alongside
|
||||
# other expert weights.
|
||||
|
||||
@@ -40,6 +40,7 @@ from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
|
||||
select_mxfp8_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
NvFp4MoeBackend,
|
||||
convert_to_nvfp4_moe_kernel_format,
|
||||
is_global_sf_supported_for_nvfp4_backend,
|
||||
make_nvfp4_moe_kernel,
|
||||
@@ -100,7 +101,11 @@ from vllm.model_executor.parameter import (
|
||||
ModelWeightParameter,
|
||||
PerTensorScaleParameter,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
from vllm.model_executor.utils import (
|
||||
is_weights_pre_processed,
|
||||
replace_parameter,
|
||||
set_weight_attrs,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -819,6 +824,8 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
quant_config: NVFP4 Quant Config
|
||||
"""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
quant_config: ModelOptNvFp4Config,
|
||||
@@ -969,6 +976,15 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
"""
|
||||
Convert NVFP4 MoE weights into kernel format and setup the kernel.
|
||||
"""
|
||||
if is_weights_pre_processed():
|
||||
if self.nvfp4_backend != NvFp4MoeBackend.FLASHINFER_TRTLLM:
|
||||
raise RuntimeError(
|
||||
"pre-processed weights require FLASHINFER_TRTLLM backend, "
|
||||
f"moe backend, got {self.nvfp4_backend}"
|
||||
)
|
||||
self._restore_padded_moe_dims(layer)
|
||||
self._build_moe_kernel(layer)
|
||||
return
|
||||
|
||||
# Use a single gscale for w13.
|
||||
if self.moe.is_act_and_mul and not torch.allclose(
|
||||
@@ -1013,7 +1029,10 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
replace_parameter(layer, "w2_weight_scale_2", w2_scale_2)
|
||||
replace_parameter(layer, "w2_input_scale", a2_scale)
|
||||
|
||||
# Setup modular kernel.
|
||||
self._build_moe_kernel(layer)
|
||||
|
||||
def _build_moe_kernel(self, layer: RoutedExperts) -> None:
|
||||
"""Build the modular MoE kernel from the (already in-format) weights."""
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
assert self.experts_cls is not None
|
||||
self.moe_kernel = make_nvfp4_moe_kernel(
|
||||
@@ -1025,6 +1044,16 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def _restore_padded_moe_dims(self, layer: RoutedExperts) -> None:
|
||||
"""Recover the padded ``moe_config`` dims from the exported weights."""
|
||||
mc = layer.moe_config
|
||||
padded_hidden = layer.w2_weight.shape[1]
|
||||
if padded_hidden != mc.hidden_dim:
|
||||
if mc.hidden_dim_unpadded is None:
|
||||
mc.hidden_dim_unpadded = mc.hidden_dim
|
||||
mc.hidden_dim = padded_hidden
|
||||
mc.intermediate_size_per_partition = layer.w2_weight.shape[2] * 2
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
|
||||
return make_nvfp4_moe_quant_config(
|
||||
backend=self.nvfp4_backend,
|
||||
@@ -2438,6 +2467,12 @@ class ModelOptLinearMethod(LinearMethodBase):
|
||||
# NVFP4 methods.
|
||||
self.kernel: Any = None
|
||||
|
||||
@property
|
||||
def supports_pre_processed_weights(self) -> bool: # type: ignore[override]
|
||||
# TODO(Isotr0py): support fp8/mxfp8 ModelOpt kernels transpose/repack.
|
||||
w = self.spec.weight
|
||||
return isinstance(w, QuantKey) and w.dtype == FP4_DTYPE
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer,
|
||||
@@ -2476,6 +2511,8 @@ class ModelOptLinearMethod(LinearMethodBase):
|
||||
expose_input_quant_key(layer, self.kernel)
|
||||
|
||||
def process_weights_after_loading(self, layer) -> None:
|
||||
if is_weights_pre_processed():
|
||||
return
|
||||
self.fmt.pre_process(layer)
|
||||
self.wkey.process(layer, WEIGHT)
|
||||
if self.akey:
|
||||
|
||||
@@ -36,7 +36,11 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
from vllm.model_executor.utils import (
|
||||
is_weights_pre_processed,
|
||||
replace_parameter,
|
||||
set_weight_attrs,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -474,6 +478,8 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
"""MXFP4 MoE quantization method."""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
def __init__(self, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
|
||||
@@ -760,9 +766,11 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
replace_parameter(layer, "w2_bias", w2_bias)
|
||||
|
||||
# Build quant config
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
self._build_moe_kernel(layer)
|
||||
|
||||
# Build kernel (modular or monolithic)
|
||||
def _build_moe_kernel(self, layer: RoutedExperts) -> None:
|
||||
"""Build the modular MoE kernel from the (already in-format) weights."""
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
if self.moe_quant_config is not None and self.experts_cls is not None:
|
||||
self.moe_kernel = make_mxfp4_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
@@ -774,6 +782,18 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
if self.mxfp4_backend == Mxfp4MoeBackend.NONE:
|
||||
return
|
||||
|
||||
if is_weights_pre_processed():
|
||||
if self.mxfp4_backend != Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8:
|
||||
raise RuntimeError(
|
||||
"pre-processed weights require FLASHINFER_TRTLLM_MXFP4_MXFP8 "
|
||||
f"moe backend, got {self.mxfp4_backend}"
|
||||
)
|
||||
self._build_moe_kernel(layer)
|
||||
return
|
||||
|
||||
w13 = layer.w13_weight
|
||||
w2 = layer.w2_weight
|
||||
w13_scale = layer.w13_weight_scale
|
||||
@@ -781,9 +801,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w13_bias = getattr(layer, "w13_bias", None)
|
||||
w2_bias = getattr(layer, "w2_bias", None)
|
||||
|
||||
if self.mxfp4_backend == Mxfp4MoeBackend.NONE:
|
||||
return
|
||||
|
||||
self._setup_kernel(layer, w13, w2, w13_scale, w2_scale, w13_bias, w2_bias)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
|
||||
@@ -98,6 +98,14 @@ def pad_nvfp4_weight_for_cutlass(
|
||||
return weight, weights_padding_bytes
|
||||
|
||||
|
||||
def nvfp4_weight_padding_bytes(layer: torch.nn.Module) -> int:
|
||||
"""
|
||||
Compute K-dim padding (packed bytes) of an NVFP4 weight,
|
||||
Aligned with `pad_nvfp4_weight_for_cutlass` above.
|
||||
"""
|
||||
return layer.weight.shape[1] - layer.input_size_per_partition // 2
|
||||
|
||||
|
||||
def pad_nvfp4_activation_for_cutlass(
|
||||
x_fp4: torch.Tensor,
|
||||
weights_padding_bytes: int,
|
||||
|
||||
@@ -17,8 +17,8 @@ Engines then load from the daemons with:
|
||||
vllm serve /path/to/model --tensor-parallel-size 4 \\
|
||||
--load-format ipc_cache
|
||||
|
||||
Only tensor parallelism is supported; pipeline, data, and expert parallelism
|
||||
are rejected at launch.
|
||||
Only tensor and expert parallelism are supported; pipeline and data
|
||||
parallelism are rejected at launch.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
@@ -54,23 +54,7 @@ from vllm.model_executor.model_loader.weight_cache.protocol import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _report_ready(message: str) -> None:
|
||||
"""Write a readiness message straight to the original stderr descriptor.
|
||||
|
||||
Loading the model pulls in FlashInfer's CuTeDSL JIT compiler, which swaps
|
||||
``sys.stdout``/``sys.stderr`` for in-memory buffers while compiling kernels
|
||||
across worker threads. That save/restore races on the interpreter-global
|
||||
streams and can leave them (and vLLM's logging handler, which caches the
|
||||
original stream object) detached, silently swallowing everything logged
|
||||
afterwards -- including the daemon's readiness announcement. Since operators
|
||||
rely on that line to know the daemon is serving, write it directly to file
|
||||
descriptor 2 so it bypasses the logging machinery entirely.
|
||||
"""
|
||||
with contextlib.suppress(OSError):
|
||||
os.write(2, (message + "\n").encode())
|
||||
logger = init_logger("vllm.model_executor.model_loader.weight_cache.daemon")
|
||||
|
||||
|
||||
def export_entries(
|
||||
@@ -192,9 +176,6 @@ class WeightCacheDaemon:
|
||||
logger.info(
|
||||
"Weight cache daemon rank %d serving on %s", self.tp_rank, socket_path
|
||||
)
|
||||
print(
|
||||
f"Weight cache daemon rank {self.tp_rank} ready: serving on {socket_path}"
|
||||
)
|
||||
if ready_callback is not None:
|
||||
ready_callback()
|
||||
try:
|
||||
@@ -277,6 +258,12 @@ class WeightCacheDaemon:
|
||||
"gpu_uuid": self._gpu_uuid(),
|
||||
},
|
||||
)
|
||||
logger.info_once(
|
||||
"Weight cache daemon rank %d sent %d tensors (+%d aliases) to engine",
|
||||
self.tp_rank,
|
||||
len(self.entries),
|
||||
len(self.aliases),
|
||||
)
|
||||
|
||||
def _handle_release(self, conn: socket.socket) -> None:
|
||||
self.entries.clear()
|
||||
@@ -308,17 +295,16 @@ def _run_daemon(
|
||||
|
||||
|
||||
def _reject_unsupported_parallelism(parallel_config: ParallelConfig) -> None:
|
||||
"""Reject every parallelism mode other than tensor parallelism."""
|
||||
"""Reject parallelism modes other than tensor/expert parallelism."""
|
||||
unsupported = {
|
||||
"pipeline parallelism": parallel_config.pipeline_parallel_size > 1,
|
||||
"data parallelism": parallel_config.data_parallel_size > 1,
|
||||
"expert parallelism": parallel_config.enable_expert_parallel,
|
||||
}
|
||||
for name, enabled in unsupported.items():
|
||||
if enabled:
|
||||
raise ValueError(
|
||||
f"The weight cache daemon only supports tensor parallelism; "
|
||||
f"{name} is not supported"
|
||||
f"The weight cache daemon only supports tensor and expert "
|
||||
f"parallelism; {name} is not supported"
|
||||
)
|
||||
|
||||
|
||||
@@ -397,16 +383,11 @@ def main() -> None:
|
||||
for proc in procs:
|
||||
proc.join()
|
||||
sys.exit(max((p.exitcode or 0) for p in procs))
|
||||
logger.info_once(
|
||||
"===== Weight cache daemon READY: all %d ranks serving in %s =====",
|
||||
logger.info(
|
||||
"Weight cache daemon ready: all %d ranks serving in %s",
|
||||
tp_size,
|
||||
args.weight_cache_socket_dir or "the default socket dir",
|
||||
)
|
||||
_report_ready(
|
||||
f"===== Weight cache daemon READY: all {tp_size} rank(s) serving in "
|
||||
f"{args.weight_cache_socket_dir or 'the default socket dir'} ====="
|
||||
)
|
||||
|
||||
for proc in procs:
|
||||
proc.join()
|
||||
sys.exit(max(proc.exitcode or 0 for proc in procs))
|
||||
|
||||
@@ -117,6 +117,13 @@ def _fp8_round_trips_via_ipc(quant_config: Any) -> bool:
|
||||
IPC_QUANT_ALLOWLIST: dict[str | None, Any] = {
|
||||
None: lambda _quant_config: True, # unquantized
|
||||
"fp8": _fp8_round_trips_via_ipc,
|
||||
"modelopt_fp4": lambda _quant_config: True,
|
||||
# Kimi-K3 routed experts (mxfp4-pack): the MegaMoE experts keep their packed
|
||||
# weights and scales as plain parameters and defer the DeepGEMM transform to
|
||||
# forward(), so the exported tensors are complete. Any other mxfp4 method
|
||||
# that repacks or drops parameters still fails closed at the engine's
|
||||
# supports_pre_processed_weights guard.
|
||||
"mxfp4": lambda _quant_config: True,
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user