[Fast Start] Support fp4 (#55465)
Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -17,8 +17,8 @@ Engines then load from the daemons with:
|
||||
vllm serve /path/to/model --tensor-parallel-size 4 \\
|
||||
--load-format ipc_cache
|
||||
|
||||
Only tensor parallelism is supported; pipeline, data, and expert parallelism
|
||||
are rejected at launch.
|
||||
Only tensor and expert parallelism are supported; pipeline and data
|
||||
parallelism are rejected at launch.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
@@ -54,23 +54,7 @@ from vllm.model_executor.model_loader.weight_cache.protocol import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _report_ready(message: str) -> None:
|
||||
"""Write a readiness message straight to the original stderr descriptor.
|
||||
|
||||
Loading the model pulls in FlashInfer's CuTeDSL JIT compiler, which swaps
|
||||
``sys.stdout``/``sys.stderr`` for in-memory buffers while compiling kernels
|
||||
across worker threads. That save/restore races on the interpreter-global
|
||||
streams and can leave them (and vLLM's logging handler, which caches the
|
||||
original stream object) detached, silently swallowing everything logged
|
||||
afterwards -- including the daemon's readiness announcement. Since operators
|
||||
rely on that line to know the daemon is serving, write it directly to file
|
||||
descriptor 2 so it bypasses the logging machinery entirely.
|
||||
"""
|
||||
with contextlib.suppress(OSError):
|
||||
os.write(2, (message + "\n").encode())
|
||||
logger = init_logger("vllm.model_executor.model_loader.weight_cache.daemon")
|
||||
|
||||
|
||||
def export_entries(
|
||||
@@ -192,9 +176,6 @@ class WeightCacheDaemon:
|
||||
logger.info(
|
||||
"Weight cache daemon rank %d serving on %s", self.tp_rank, socket_path
|
||||
)
|
||||
print(
|
||||
f"Weight cache daemon rank {self.tp_rank} ready: serving on {socket_path}"
|
||||
)
|
||||
if ready_callback is not None:
|
||||
ready_callback()
|
||||
try:
|
||||
@@ -277,6 +258,12 @@ class WeightCacheDaemon:
|
||||
"gpu_uuid": self._gpu_uuid(),
|
||||
},
|
||||
)
|
||||
logger.info_once(
|
||||
"Weight cache daemon rank %d sent %d tensors (+%d aliases) to engine",
|
||||
self.tp_rank,
|
||||
len(self.entries),
|
||||
len(self.aliases),
|
||||
)
|
||||
|
||||
def _handle_release(self, conn: socket.socket) -> None:
|
||||
self.entries.clear()
|
||||
@@ -308,17 +295,16 @@ def _run_daemon(
|
||||
|
||||
|
||||
def _reject_unsupported_parallelism(parallel_config: ParallelConfig) -> None:
|
||||
"""Reject every parallelism mode other than tensor parallelism."""
|
||||
"""Reject parallelism modes other than tensor/expert parallelism."""
|
||||
unsupported = {
|
||||
"pipeline parallelism": parallel_config.pipeline_parallel_size > 1,
|
||||
"data parallelism": parallel_config.data_parallel_size > 1,
|
||||
"expert parallelism": parallel_config.enable_expert_parallel,
|
||||
}
|
||||
for name, enabled in unsupported.items():
|
||||
if enabled:
|
||||
raise ValueError(
|
||||
f"The weight cache daemon only supports tensor parallelism; "
|
||||
f"{name} is not supported"
|
||||
f"The weight cache daemon only supports tensor and expert "
|
||||
f"parallelism; {name} is not supported"
|
||||
)
|
||||
|
||||
|
||||
@@ -397,16 +383,11 @@ def main() -> None:
|
||||
for proc in procs:
|
||||
proc.join()
|
||||
sys.exit(max((p.exitcode or 0) for p in procs))
|
||||
logger.info_once(
|
||||
"===== Weight cache daemon READY: all %d ranks serving in %s =====",
|
||||
logger.info(
|
||||
"Weight cache daemon ready: all %d ranks serving in %s",
|
||||
tp_size,
|
||||
args.weight_cache_socket_dir or "the default socket dir",
|
||||
)
|
||||
_report_ready(
|
||||
f"===== Weight cache daemon READY: all {tp_size} rank(s) serving in "
|
||||
f"{args.weight_cache_socket_dir or 'the default socket dir'} ====="
|
||||
)
|
||||
|
||||
for proc in procs:
|
||||
proc.join()
|
||||
sys.exit(max(proc.exitcode or 0 for proc in procs))
|
||||
|
||||
@@ -117,6 +117,13 @@ def _fp8_round_trips_via_ipc(quant_config: Any) -> bool:
|
||||
IPC_QUANT_ALLOWLIST: dict[str | None, Any] = {
|
||||
None: lambda _quant_config: True, # unquantized
|
||||
"fp8": _fp8_round_trips_via_ipc,
|
||||
"modelopt_fp4": lambda _quant_config: True,
|
||||
# Kimi-K3 routed experts (mxfp4-pack): the MegaMoE experts keep their packed
|
||||
# weights and scales as plain parameters and defer the DeepGEMM transform to
|
||||
# forward(), so the exported tensors are complete. Any other mxfp4 method
|
||||
# that repacks or drops parameters still fails closed at the engine's
|
||||
# supports_pre_processed_weights guard.
|
||||
"mxfp4": lambda _quant_config: True,
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user