Dflash2 load fix (#53435)

This commit is contained in:
Karan Shelar
2026-08-24 21:32:43 -07:00
committed by GitHub
parent 1f9444a34f
commit a9a17e7095
2 changed files with 115 additions and 1 deletions
+112
View File
@@ -116,3 +116,115 @@ def test_selector_asks_for_fp32_proposal_logits():
assert dtype is torch.float32
assert fill == float("-inf")
@pytest.mark.skip_global_cleanup
def test_dflash2_model_decoder_layer_cls(monkeypatch):
from types import SimpleNamespace
from vllm.config import set_current_vllm_config
from vllm.model_executor.models.qwen3_dflash2 import (
DFlash2Qwen3DecoderLayer,
DFlash2Qwen3Model,
)
# 1. Mock get_current_vllm_config and TP groups
mock_current_vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(
block_size=16,
user_specified_block_size=False,
kv_cache_dtype_skip_layers=[],
cache_dtype="auto",
sliding_window=None,
enable_prefix_caching=False,
),
kv_transfer_config=None,
speculative_config=None,
attention_config=SimpleNamespace(
use_non_causal=False,
backend=None,
backend_per_kind={},
),
parallel_config=SimpleNamespace(
prefill_context_parallel_size=1,
decode_context_parallel_size=1,
),
compilation_config=SimpleNamespace(
compile_custom_ops=False,
custom_ops="all",
enabled_custom_ops=set(),
static_forward_context={},
mode=0, # CompilationMode.NONE is 0
),
model_config=SimpleNamespace(
dtype=torch.float32,
is_mm_prefix_lm=False,
),
)
from vllm.platforms import current_platform
monkeypatch.setattr(
current_platform,
"get_attn_backend_cls",
lambda *args, **kwargs: (
"vllm.v1.attention.backends.cpu_attn.CPUAttentionBackend"
),
)
class MockGroup:
rank_in_group = 0
world_size = 1
monkeypatch.setattr(
"vllm.distributed.parallel_state._TP",
MockGroup(),
)
# 2. Mock vllm_config
hf_config = SimpleNamespace(
vocab_size=1000,
hidden_size=256,
num_hidden_layers=2,
num_attention_heads=8,
num_key_value_heads=2,
max_position_embeddings=2048,
rms_norm_eps=1e-6,
rope_parameters={},
intermediate_size=512,
hidden_act="silu",
dflash_config={
"selector_rank": 4,
"selector_top_k": 3,
"conv_kernel_size": 3,
"conv_group_size": 2,
"use_aux_hidden_state": False,
},
)
vllm_config = SimpleNamespace(
speculative_config=SimpleNamespace(
draft_model_config=SimpleNamespace(
hf_config=hf_config,
quantization=None,
),
num_speculative_tokens=4,
enable_adaptive_verification=False,
),
model_config=SimpleNamespace(
dtype=torch.float32,
is_mm_prefix_lm=False,
),
load_config=SimpleNamespace(
quantization=None,
quantization_param_path=None,
),
)
mock_current_vllm_config.speculative_config = vllm_config.speculative_config
vllm_config.compilation_config = mock_current_vllm_config.compilation_config
# 3. Instantiate the model under meta device to avoid parameter allocation issues
with set_current_vllm_config(mock_current_vllm_config), torch.device("meta"):
model = DFlash2Qwen3Model(vllm_config=vllm_config)
# 4. Assert that the layers are DFlash2Qwen3DecoderLayer (the subclass)
assert len(model.layers) == 2
assert isinstance(model.layers[0], DFlash2Qwen3DecoderLayer)
+3 -1
View File
@@ -377,6 +377,8 @@ class DFlashQwen3DecoderLayer(nn.Module):
@support_torch_compile
class DFlashQwen3Model(nn.Module):
decoder_layer_cls = DFlashQwen3DecoderLayer
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={
"midlayer.": "layers.0.",
@@ -439,7 +441,7 @@ class DFlashQwen3Model(nn.Module):
self.layers = nn.ModuleList(
[
DFlashQwen3DecoderLayer(
self.decoder_layer_cls(
current_vllm_config,
config=self.config,
layer_idx=layer_idx,