[CI] Fix various failures on main (#47197)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -37,23 +37,18 @@ from vllm.platforms import current_platform
|
|||||||
from vllm.utils.network_utils import get_open_port
|
from vllm.utils.network_utils import get_open_port
|
||||||
|
|
||||||
|
|
||||||
def _weight_transfer_ray_env_vars() -> dict[str, str]:
|
|
||||||
if not current_platform.is_rocm():
|
|
||||||
return {}
|
|
||||||
|
|
||||||
return {
|
|
||||||
"RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES": "1",
|
|
||||||
"RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES": "1",
|
|
||||||
"RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES": "1",
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _init_ray_for_weight_transfer() -> None:
|
def _init_ray_for_weight_transfer() -> None:
|
||||||
if ray.is_initialized():
|
if ray.is_initialized():
|
||||||
return
|
return
|
||||||
ray.init(
|
ray.init(
|
||||||
ignore_reinit_error=True,
|
ignore_reinit_error=True,
|
||||||
runtime_env={"env_vars": _weight_transfer_ray_env_vars()},
|
runtime_env={
|
||||||
|
"env_vars": {
|
||||||
|
"RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES": "1",
|
||||||
|
"RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES": "1",
|
||||||
|
"RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES": "1",
|
||||||
|
}
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -66,7 +61,7 @@ def _get_ray_assigned_device() -> torch.device:
|
|||||||
|
|
||||||
def _set_ray_assigned_device() -> torch.device:
|
def _set_ray_assigned_device() -> torch.device:
|
||||||
device = _get_ray_assigned_device()
|
device = _get_ray_assigned_device()
|
||||||
torch.accelerator.set_device_index(device)
|
current_platform.set_device(device)
|
||||||
return device
|
return device
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -403,6 +403,7 @@ def _run_ref_mamba_state_worker():
|
|||||||
GPUModelRunner._sample = fake_sample_fn
|
GPUModelRunner._sample = fake_sample_fn
|
||||||
engine = LLM(
|
engine = LLM(
|
||||||
model=MODEL,
|
model=MODEL,
|
||||||
|
load_format="dummy",
|
||||||
block_size=BLOCK_SIZE,
|
block_size=BLOCK_SIZE,
|
||||||
hf_overrides={"num_hidden_layers": NUM_HIDDEN_LAYERS},
|
hf_overrides={"num_hidden_layers": NUM_HIDDEN_LAYERS},
|
||||||
seed=42,
|
seed=42,
|
||||||
@@ -750,6 +751,7 @@ def test_mamba_prefix_cache_mrv1(monkeypatch: pytest.MonkeyPatch):
|
|||||||
|
|
||||||
engine = LLM(
|
engine = LLM(
|
||||||
model=MODEL,
|
model=MODEL,
|
||||||
|
load_format="dummy",
|
||||||
enable_prefix_caching=True,
|
enable_prefix_caching=True,
|
||||||
block_size=BLOCK_SIZE,
|
block_size=BLOCK_SIZE,
|
||||||
mamba_cache_mode="align",
|
mamba_cache_mode="align",
|
||||||
|
|||||||
@@ -907,7 +907,7 @@ class RoutedExperts(PluggableLayer):
|
|||||||
# Unified loading logic for fused and non-fused experts
|
# Unified loading logic for fused and non-fused experts
|
||||||
loaded_experts = experts_shard.unbind()
|
loaded_experts = experts_shard.unbind()
|
||||||
for expert_id, loaded_expert in enumerate(loaded_experts, start=start):
|
for expert_id, loaded_expert in enumerate(loaded_experts, start=start):
|
||||||
success = self.weight_loader(
|
success = param.weight_loader(
|
||||||
param=param,
|
param=param,
|
||||||
loaded_weight=loaded_expert,
|
loaded_weight=loaded_expert,
|
||||||
weight_name=weight_name,
|
weight_name=weight_name,
|
||||||
|
|||||||
@@ -1445,6 +1445,8 @@ class DeepseekV2Model(nn.Module):
|
|||||||
hidden_states, residual = combined_states.split(
|
hidden_states, residual = combined_states.split(
|
||||||
[self.hidden_size, self.hidden_size], dim=-1
|
[self.hidden_size, self.hidden_size], dim=-1
|
||||||
)
|
)
|
||||||
|
# fused_add_rms_norm requires a contiguous residual
|
||||||
|
residual = residual.contiguous()
|
||||||
if idx in self.aux_hidden_state_layers:
|
if idx in self.aux_hidden_state_layers:
|
||||||
aux_hidden_state = hidden_states + residual
|
aux_hidden_state = hidden_states + residual
|
||||||
if aux_hidden_state.shape[0] != positions.shape[0]:
|
if aux_hidden_state.shape[0] != positions.shape[0]:
|
||||||
@@ -1469,6 +1471,8 @@ class DeepseekV2Model(nn.Module):
|
|||||||
hidden_states, residual = combined_states.split(
|
hidden_states, residual = combined_states.split(
|
||||||
[self.hidden_size, self.hidden_size], dim=-1
|
[self.hidden_size, self.hidden_size], dim=-1
|
||||||
)
|
)
|
||||||
|
# fused_add_rms_norm requires a contiguous residual
|
||||||
|
residual = residual.contiguous()
|
||||||
|
|
||||||
if self.end_layer in self.aux_hidden_state_layers:
|
if self.end_layer in self.aux_hidden_state_layers:
|
||||||
aux_hidden_states.append(hidden_states + residual)
|
aux_hidden_states.append(hidden_states + residual)
|
||||||
|
|||||||
@@ -761,6 +761,7 @@ class Gemma3ForConditionalGeneration(
|
|||||||
max_frames_per_batch: int,
|
max_frames_per_batch: int,
|
||||||
device: torch.device,
|
device: torch.device,
|
||||||
dtype: torch.dtype,
|
dtype: torch.dtype,
|
||||||
|
path: str = "default",
|
||||||
):
|
):
|
||||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||||
EncoderCudaGraphCaptureInputs,
|
EncoderCudaGraphCaptureInputs,
|
||||||
@@ -792,6 +793,7 @@ class Gemma3ForConditionalGeneration(
|
|||||||
mm_kwargs: dict[str, Any],
|
mm_kwargs: dict[str, Any],
|
||||||
max_batch_size: int,
|
max_batch_size: int,
|
||||||
max_frames_per_batch: int,
|
max_frames_per_batch: int,
|
||||||
|
path: str = "default",
|
||||||
):
|
):
|
||||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||||
EncoderCudaGraphReplayBuffers,
|
EncoderCudaGraphReplayBuffers,
|
||||||
@@ -804,6 +806,7 @@ class Gemma3ForConditionalGeneration(
|
|||||||
def encoder_cudagraph_forward(
|
def encoder_cudagraph_forward(
|
||||||
self,
|
self,
|
||||||
values: dict[str, torch.Tensor],
|
values: dict[str, torch.Tensor],
|
||||||
|
path: str = "default",
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
pixel_values = values["pixel_values"]
|
pixel_values = values["pixel_values"]
|
||||||
image_features = self.vision_tower(pixel_values)
|
image_features = self.vision_tower(pixel_values)
|
||||||
|
|||||||
Reference in New Issue
Block a user