Dflash2 load fix (#53435)
This commit is contained in:
@@ -116,3 +116,115 @@ def test_selector_asks_for_fp32_proposal_logits():
|
||||
|
||||
assert dtype is torch.float32
|
||||
assert fill == float("-inf")
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_dflash2_model_decoder_layer_cls(monkeypatch):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.model_executor.models.qwen3_dflash2 import (
|
||||
DFlash2Qwen3DecoderLayer,
|
||||
DFlash2Qwen3Model,
|
||||
)
|
||||
|
||||
# 1. Mock get_current_vllm_config and TP groups
|
||||
mock_current_vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
block_size=16,
|
||||
user_specified_block_size=False,
|
||||
kv_cache_dtype_skip_layers=[],
|
||||
cache_dtype="auto",
|
||||
sliding_window=None,
|
||||
enable_prefix_caching=False,
|
||||
),
|
||||
kv_transfer_config=None,
|
||||
speculative_config=None,
|
||||
attention_config=SimpleNamespace(
|
||||
use_non_causal=False,
|
||||
backend=None,
|
||||
backend_per_kind={},
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
prefill_context_parallel_size=1,
|
||||
decode_context_parallel_size=1,
|
||||
),
|
||||
compilation_config=SimpleNamespace(
|
||||
compile_custom_ops=False,
|
||||
custom_ops="all",
|
||||
enabled_custom_ops=set(),
|
||||
static_forward_context={},
|
||||
mode=0, # CompilationMode.NONE is 0
|
||||
),
|
||||
model_config=SimpleNamespace(
|
||||
dtype=torch.float32,
|
||||
is_mm_prefix_lm=False,
|
||||
),
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
monkeypatch.setattr(
|
||||
current_platform,
|
||||
"get_attn_backend_cls",
|
||||
lambda *args, **kwargs: (
|
||||
"vllm.v1.attention.backends.cpu_attn.CPUAttentionBackend"
|
||||
),
|
||||
)
|
||||
|
||||
class MockGroup:
|
||||
rank_in_group = 0
|
||||
world_size = 1
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.parallel_state._TP",
|
||||
MockGroup(),
|
||||
)
|
||||
|
||||
# 2. Mock vllm_config
|
||||
hf_config = SimpleNamespace(
|
||||
vocab_size=1000,
|
||||
hidden_size=256,
|
||||
num_hidden_layers=2,
|
||||
num_attention_heads=8,
|
||||
num_key_value_heads=2,
|
||||
max_position_embeddings=2048,
|
||||
rms_norm_eps=1e-6,
|
||||
rope_parameters={},
|
||||
intermediate_size=512,
|
||||
hidden_act="silu",
|
||||
dflash_config={
|
||||
"selector_rank": 4,
|
||||
"selector_top_k": 3,
|
||||
"conv_kernel_size": 3,
|
||||
"conv_group_size": 2,
|
||||
"use_aux_hidden_state": False,
|
||||
},
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(
|
||||
draft_model_config=SimpleNamespace(
|
||||
hf_config=hf_config,
|
||||
quantization=None,
|
||||
),
|
||||
num_speculative_tokens=4,
|
||||
enable_adaptive_verification=False,
|
||||
),
|
||||
model_config=SimpleNamespace(
|
||||
dtype=torch.float32,
|
||||
is_mm_prefix_lm=False,
|
||||
),
|
||||
load_config=SimpleNamespace(
|
||||
quantization=None,
|
||||
quantization_param_path=None,
|
||||
),
|
||||
)
|
||||
mock_current_vllm_config.speculative_config = vllm_config.speculative_config
|
||||
vllm_config.compilation_config = mock_current_vllm_config.compilation_config
|
||||
|
||||
# 3. Instantiate the model under meta device to avoid parameter allocation issues
|
||||
with set_current_vllm_config(mock_current_vllm_config), torch.device("meta"):
|
||||
model = DFlash2Qwen3Model(vllm_config=vllm_config)
|
||||
|
||||
# 4. Assert that the layers are DFlash2Qwen3DecoderLayer (the subclass)
|
||||
assert len(model.layers) == 2
|
||||
assert isinstance(model.layers[0], DFlash2Qwen3DecoderLayer)
|
||||
|
||||
@@ -377,6 +377,8 @@ class DFlashQwen3DecoderLayer(nn.Module):
|
||||
|
||||
@support_torch_compile
|
||||
class DFlashQwen3Model(nn.Module):
|
||||
decoder_layer_cls = DFlashQwen3DecoderLayer
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
"midlayer.": "layers.0.",
|
||||
@@ -439,7 +441,7 @@ class DFlashQwen3Model(nn.Module):
|
||||
|
||||
self.layers = nn.ModuleList(
|
||||
[
|
||||
DFlashQwen3DecoderLayer(
|
||||
self.decoder_layer_cls(
|
||||
current_vllm_config,
|
||||
config=self.config,
|
||||
layer_idx=layer_idx,
|
||||
|
||||
Reference in New Issue
Block a user