Merge remote-tracking branch 'origin/main' into sync/lesj/qwen4exp-bugfixes
Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Any
|
||||
|
||||
@@ -61,6 +62,21 @@ class MHCPreNormKernel(VllmJitKernel["MHCPreNormKernel.CompileKey"]):
|
||||
**asdict(compile_key)
|
||||
)
|
||||
|
||||
def compile_many(self, compile_keys: Iterable[CompileKey]) -> None:
|
||||
missing = list(
|
||||
dict.fromkeys(
|
||||
key for key in compile_keys if key not in self._compiled_cache
|
||||
)
|
||||
)
|
||||
if len(missing) < 2:
|
||||
return super().compile_many(missing)
|
||||
|
||||
# TileLang elaborates serially before compiling independent TIR modules.
|
||||
compiled = self.kernel.par_compile(
|
||||
[asdict(key) for key in missing], num_workers=min(4, len(missing))
|
||||
)
|
||||
self._compiled_cache.update(zip(missing, compiled, strict=True))
|
||||
|
||||
def __call__(self, *tensors, **fields):
|
||||
compile_key = self.dispatch(
|
||||
n_splits=tensors[0].shape[0],
|
||||
|
||||
@@ -17,8 +17,30 @@ Engines then load from the daemons with:
|
||||
vllm serve /path/to/model --tensor-parallel-size 4 \\
|
||||
--load-format ipc_cache
|
||||
|
||||
Only tensor and expert parallelism are supported; pipeline and data
|
||||
parallelism are rejected at launch.
|
||||
Only tensor and expert parallelism are supported; pipeline and data parallelism
|
||||
are rejected at launch.
|
||||
|
||||
For multi-node tensor parallelism, run one launcher per node with a shared
|
||||
rendezvous so the global TP group forms across nodes (CUDA IPC handles are
|
||||
node-local, so each node serves only its local GPUs' shards). Reuse the same
|
||||
``--nnodes``/``--node-rank``/``--master-addr`` flags you pass the engine, plus a
|
||||
``--weight-cache-master-port`` distinct from the engine's ``--master-port``:
|
||||
|
||||
# node 0 (8 local GPUs)
|
||||
python -m vllm.model_executor.model_loader.weight_cache.daemon \\
|
||||
--model /path/to/model --tensor-parallel-size 16 \\
|
||||
--nnodes 2 --node-rank 0 --master-addr 10.0.0.1 \\
|
||||
--weight-cache-master-port 29600
|
||||
# node 1 (8 local GPUs)
|
||||
python -m vllm.model_executor.model_loader.weight_cache.daemon \\
|
||||
--model /path/to/model --tensor-parallel-size 16 \\
|
||||
--nnodes 2 --node-rank 1 --master-addr 10.0.0.1 \\
|
||||
--weight-cache-master-port 29600
|
||||
|
||||
The global TP rank of local GPU ``i`` on node ``r`` is
|
||||
``r * (tp_size // nnodes) + i``, matching vLLM's contiguous per-node rank
|
||||
assignment, so each engine worker maps its shard from the daemon on its own
|
||||
node.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
@@ -132,11 +154,13 @@ class WeightCacheDaemon:
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
tp_rank: int,
|
||||
local_rank: int,
|
||||
distributed_init_method: str,
|
||||
socket_dir: str | None = None,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
self.tp_rank = tp_rank
|
||||
self.local_rank = local_rank
|
||||
self.distributed_init_method = distributed_init_method
|
||||
self.socket_dir = socket_dir
|
||||
self.model: torch.nn.Module | None = None
|
||||
@@ -152,12 +176,12 @@ class WeightCacheDaemon:
|
||||
|
||||
def load_model(self) -> None:
|
||||
tp_size = self.cache_config.tp_size
|
||||
torch.accelerator.set_device_index(self.tp_rank)
|
||||
torch.accelerator.set_device_index(self.local_rank)
|
||||
init_distributed_environment(
|
||||
world_size=tp_size,
|
||||
rank=self.tp_rank,
|
||||
distributed_init_method=self.distributed_init_method,
|
||||
local_rank=self.tp_rank,
|
||||
local_rank=self.local_rank,
|
||||
backend=current_platform.dist_backend,
|
||||
)
|
||||
with set_current_vllm_config(self.vllm_config):
|
||||
@@ -307,13 +331,14 @@ class WeightCacheDaemon:
|
||||
|
||||
def _run_daemon(
|
||||
tp_rank: int,
|
||||
local_rank: int,
|
||||
vllm_config: VllmConfig,
|
||||
distributed_init_method: str,
|
||||
socket_dir: str | None,
|
||||
ready_queue: "multiprocessing.Queue[int]",
|
||||
) -> None:
|
||||
daemon = WeightCacheDaemon(
|
||||
vllm_config, tp_rank, distributed_init_method, socket_dir
|
||||
vllm_config, tp_rank, local_rank, distributed_init_method, socket_dir
|
||||
)
|
||||
daemon.load_model()
|
||||
daemon.serve_forever(ready_callback=lambda: ready_queue.put(tp_rank))
|
||||
@@ -348,6 +373,15 @@ def main() -> None:
|
||||
default=None,
|
||||
help="Directory for the daemon Unix sockets (default: tempdir).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--weight-cache-master-port",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Rendezvous port for the daemon's own TP group. Must differ from "
|
||||
"the engine's --master-port (the daemon holds its group open while "
|
||||
"serving) and match across nodes. Required when --nnodes > 1; defaults "
|
||||
"to a free port for single-node.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
engine_args = EngineArgs.from_cli_args(args)
|
||||
vllm_config = engine_args.create_engine_config()
|
||||
@@ -363,22 +397,41 @@ def main() -> None:
|
||||
_reject_unsupported_parallelism(parallel_config)
|
||||
tp_size = parallel_config.tensor_parallel_size
|
||||
|
||||
distributed_init_method = get_distributed_init_method("127.0.0.1", get_open_port())
|
||||
nnodes = parallel_config.nnodes
|
||||
node_rank = parallel_config.node_rank
|
||||
local_world_size = tp_size // nnodes
|
||||
|
||||
# The daemon forms its own TP group and holds it open while serving, so it
|
||||
# needs a rendezvous port distinct from the engine's --master-port. All
|
||||
# nodes must agree on it; single-node can auto-pick a free port. The master
|
||||
# address is reused from the engine's --master-addr (node_rank 0).
|
||||
master_addr = parallel_config.master_addr
|
||||
if nnodes > 1 and args.weight_cache_master_port is None:
|
||||
raise ValueError("--weight-cache-master-port is required when --nnodes > 1")
|
||||
master_port = args.weight_cache_master_port or get_open_port()
|
||||
distributed_init_method = get_distributed_init_method(master_addr, master_port)
|
||||
|
||||
ctx = multiprocessing.get_context("spawn")
|
||||
ready_queue: multiprocessing.Queue[int] = ctx.Queue()
|
||||
# Global TP rank of local GPU i on this node; local index == device index.
|
||||
global_ranks = [
|
||||
node_rank * local_world_size + local_rank
|
||||
for local_rank in range(local_world_size)
|
||||
]
|
||||
procs = [
|
||||
ctx.Process(
|
||||
target=_run_daemon,
|
||||
args=(
|
||||
rank,
|
||||
global_rank,
|
||||
local_rank,
|
||||
vllm_config,
|
||||
distributed_init_method,
|
||||
args.weight_cache_socket_dir,
|
||||
ready_queue,
|
||||
),
|
||||
name=f"vllm-weight-cache-daemon-{rank}",
|
||||
name=f"vllm-weight-cache-daemon-{global_rank}",
|
||||
)
|
||||
for rank in range(tp_size)
|
||||
for local_rank, global_rank in enumerate(global_ranks)
|
||||
]
|
||||
for proc in procs:
|
||||
proc.start()
|
||||
@@ -391,7 +444,7 @@ def main() -> None:
|
||||
signal.signal(signal.SIGTERM, _shutdown)
|
||||
|
||||
ready_ranks: set[int] = set()
|
||||
while len(ready_ranks) < tp_size:
|
||||
while len(ready_ranks) < local_world_size:
|
||||
try:
|
||||
ready_ranks.add(ready_queue.get(timeout=1.0))
|
||||
except queue.Empty:
|
||||
@@ -407,11 +460,16 @@ def main() -> None:
|
||||
for proc in procs:
|
||||
proc.join()
|
||||
sys.exit(max((p.exitcode or 0) for p in procs))
|
||||
logger.info(
|
||||
"Weight cache daemon ready: all %d ranks serving in %s",
|
||||
tp_size,
|
||||
args.weight_cache_socket_dir or "the default socket dir",
|
||||
socket_dir_msg = args.weight_cache_socket_dir or "the default socket dir"
|
||||
logger.info_once(
|
||||
"===== Weight cache daemon READY: node %d/%d serving %d local rank(s) "
|
||||
"in %s =====",
|
||||
node_rank,
|
||||
nnodes,
|
||||
local_world_size,
|
||||
socket_dir_msg,
|
||||
)
|
||||
|
||||
for proc in procs:
|
||||
proc.join()
|
||||
sys.exit(max(proc.exitcode or 0 for proc in procs))
|
||||
|
||||
@@ -963,14 +963,18 @@ class VllmJitKernel(Generic[CompileKeyT], ABC):
|
||||
"""Compile one warmup key."""
|
||||
raise NotImplementedError
|
||||
|
||||
def compile_many(self, compile_keys: Iterable[CompileKeyT]) -> None:
|
||||
"""Compile a batch of warmup keys, allowing backend-specific scheduling."""
|
||||
for compile_key in compile_keys:
|
||||
self.compile(compile_key)
|
||||
|
||||
def register_warmup(self, *args: Any, **kwargs: Any) -> None:
|
||||
"""Register this kernel with the active runner's warmup registry."""
|
||||
JitWarmupRegistry.register(self, *args, **kwargs)
|
||||
|
||||
def warmup(self, *args: Any, **kwargs: Any) -> None:
|
||||
"""Compile this kernel's warmup keys."""
|
||||
for compile_key in self.get_warmup_keys(*args, **kwargs):
|
||||
self.compile(compile_key)
|
||||
self.compile_many(self.get_warmup_keys(*args, **kwargs))
|
||||
|
||||
|
||||
def _same_value(left: Any, right: Any) -> bool:
|
||||
@@ -1111,5 +1115,4 @@ class JitWarmupRegistry:
|
||||
f"{kernel.__class__.__name__} ({len(compile_keys)} keys)",
|
||||
refresh=False,
|
||||
)
|
||||
for compile_key in compile_keys:
|
||||
kernel.compile(compile_key)
|
||||
kernel.compile_many(compile_keys)
|
||||
|
||||
Reference in New Issue
Block a user