[Qwen3.8-Flash-Next] Remove torch.compile for NVIDIA implementation (#55272)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
@@ -147,9 +147,9 @@ def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
|
||||
model_state.uses_ngram_embedding = True
|
||||
model_state.ngram_context_len = 3
|
||||
model_state.ngram_eos_token_id = 99
|
||||
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
|
||||
model_state.ngram_context = torch.empty((8, 3), dtype=torch.int32)
|
||||
model_state.ngram_context_offsets = torch.arange(-3, 0, dtype=torch.int64)
|
||||
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
|
||||
model_state.ple_query_start_loc = torch.empty(9, dtype=torch.int32)
|
||||
|
||||
input_batch = SimpleNamespace(
|
||||
num_reqs=2,
|
||||
@@ -167,34 +167,57 @@ def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
|
||||
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
|
||||
model_inputs = model_state.prepare_inputs(input_batch, req_states)
|
||||
|
||||
expected_query_start_loc = torch.full((9,), 3, dtype=torch.int32)
|
||||
expected_query_start_loc[0] = 0
|
||||
expected_query_start_loc[1] = 2
|
||||
torch.testing.assert_close(
|
||||
model_inputs["query_start_loc"],
|
||||
torch.tensor([0, 2, 3, 3], dtype=torch.int32),
|
||||
model_inputs["query_start_loc"], expected_query_start_loc
|
||||
)
|
||||
expected_context = torch.full((8, 3), 99, dtype=torch.int32)
|
||||
expected_context[:2] = torch.tensor([[99, 99, 20], [1, 2, 3]])
|
||||
torch.testing.assert_close(model_inputs["ngram_context"], expected_context)
|
||||
|
||||
# Retain the views to detect reallocations as the request layout changes.
|
||||
query_start_loc = model_inputs["query_start_loc"]
|
||||
ngram_context = model_inputs["ngram_context"]
|
||||
input_batch.num_reqs = 1
|
||||
input_batch.num_reqs_after_padding = 1
|
||||
input_batch.idx_mapping = torch.tensor([0])
|
||||
input_batch.query_start_loc = torch.tensor([0, 3], dtype=torch.int32)
|
||||
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
|
||||
model_inputs = model_state.prepare_inputs(input_batch, req_states)
|
||||
|
||||
expected_query_start_loc.fill_(3)
|
||||
expected_query_start_loc[0] = 0
|
||||
torch.testing.assert_close(
|
||||
model_inputs["ngram_context"],
|
||||
torch.tensor([[99, 99, 20], [1, 2, 3], [99, 99, 99]], dtype=torch.int32),
|
||||
model_inputs["query_start_loc"], expected_query_start_loc
|
||||
)
|
||||
expected_context.fill_(99)
|
||||
expected_context[0] = torch.tensor([1, 2, 3])
|
||||
torch.testing.assert_close(model_inputs["ngram_context"], expected_context)
|
||||
assert model_inputs["query_start_loc"].data_ptr() == query_start_loc.data_ptr()
|
||||
assert model_inputs["ngram_context"].data_ptr() == ngram_context.data_ptr()
|
||||
|
||||
|
||||
def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
|
||||
model_state = object.__new__(Qwen4ExpModelState)
|
||||
model_state.uses_ngram_embedding = True
|
||||
model_state.ngram_eos_token_id = 99
|
||||
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
|
||||
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
|
||||
model_state.ngram_context = torch.empty((8, 3), dtype=torch.int32)
|
||||
model_state.ple_query_start_loc = torch.empty(9, dtype=torch.int32)
|
||||
|
||||
with patch.object(MambaHybridModelState, "prepare_dummy_inputs", return_value={}):
|
||||
first = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
|
||||
first = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=4)
|
||||
# Dummy runs establish the addresses used during CUDA graph capture.
|
||||
query_start_loc_ptr = first["query_start_loc"].data_ptr()
|
||||
ngram_context_ptr = first["ngram_context"].data_ptr()
|
||||
second = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
|
||||
second = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=4)
|
||||
|
||||
expected_query_start_loc = torch.full((9,), 4, dtype=torch.int32)
|
||||
expected_query_start_loc[:4] = torch.tensor([0, 1, 2, 4], dtype=torch.int32)
|
||||
torch.testing.assert_close(second["query_start_loc"], expected_query_start_loc)
|
||||
torch.testing.assert_close(
|
||||
second["query_start_loc"], torch.tensor([0, 2, 5, 8], dtype=torch.int32)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
second["ngram_context"], torch.full((3, 3), 99, dtype=torch.int32)
|
||||
second["ngram_context"], torch.full((8, 3), 99, dtype=torch.int32)
|
||||
)
|
||||
assert second["query_start_loc"].data_ptr() == query_start_loc_ptr
|
||||
assert second["ngram_context"].data_ptr() == ngram_context_ptr
|
||||
|
||||
@@ -498,6 +498,17 @@ def _ngram_hash_params(device: torch.device, context_len: int) -> dict:
|
||||
([3], [1], [[11]], 20),
|
||||
([4, 4], [0, 7], [[11, 12], [13, 14]], 20),
|
||||
([4, 0, 3], [], [[11, 12], [13, 14], [15, 16]], 20),
|
||||
(
|
||||
[3, 2, 0, 0],
|
||||
[],
|
||||
[
|
||||
[11, 12],
|
||||
[13, 14],
|
||||
[_NGRAM_EOS_TOKEN_ID, _NGRAM_EOS_TOKEN_ID],
|
||||
[_NGRAM_EOS_TOKEN_ID, _NGRAM_EOS_TOKEN_ID],
|
||||
],
|
||||
20,
|
||||
),
|
||||
(
|
||||
[1, 33, 2],
|
||||
[5, 32],
|
||||
@@ -528,6 +539,7 @@ def _ngram_hash_params(device: torch.device, context_len: int) -> dict:
|
||||
"bigram-only",
|
||||
"power-of-two",
|
||||
"empty-request",
|
||||
"trailing-padded-requests",
|
||||
"three-requests",
|
||||
"six-requests",
|
||||
"four-gram",
|
||||
|
||||
@@ -45,7 +45,6 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
|
||||
indexer = SimpleNamespace(
|
||||
skip_topk=True,
|
||||
_metadata=lambda: (raw_metadata, compressed_metadata),
|
||||
index_qk_proj=lambda hidden: (torch.zeros(2, 2), None),
|
||||
index_n_heads=1,
|
||||
index_kv_heads=1,
|
||||
index_head_dim=1,
|
||||
@@ -80,7 +79,7 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
|
||||
|
||||
actual = indexer_qsa.QSAIndexer.forward(
|
||||
indexer,
|
||||
torch.zeros(2, 4),
|
||||
torch.zeros(2, 2),
|
||||
torch.tensor([7, 8]),
|
||||
rows,
|
||||
)
|
||||
@@ -446,6 +445,71 @@ def test_qsa_compressed_metadata_keeps_dummy_slots_inert() -> None:
|
||||
assert metadata.k_work_metadata.tolist() == [[0, 0], [2, 0], [2, 1], [-1, -1]]
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_qsa_unfused_cache_update_ignores_padded_qk() -> None:
|
||||
"""Padded projected Q/K rows must not affect either side cache."""
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
|
||||
device = torch.device("cuda")
|
||||
# Five tokens complete one compressed group and retain four keys in the ring.
|
||||
raw_metadata = SimpleNamespace(
|
||||
num_actual_tokens=5,
|
||||
slot_mapping=torch.tensor([-1, 1, 2, 3, 0], device=device),
|
||||
block_table=torch.zeros((1, 1), dtype=torch.int32, device=device),
|
||||
token_to_req=torch.zeros(5, dtype=torch.int32, device=device),
|
||||
query_start_loc=torch.tensor([0, 5], dtype=torch.int32, device=device),
|
||||
logical_positions=torch.arange(5, device=device),
|
||||
)
|
||||
compressed_metadata = SimpleNamespace(
|
||||
slot_mapping=torch.tensor([-1, -1, -1, 0, -1], device=device),
|
||||
)
|
||||
with torch.device(device):
|
||||
rope = get_rope(
|
||||
head_size=128,
|
||||
max_position=32,
|
||||
rope_parameters={"rope_type": "default", "partial_rotary_factor": 0.5},
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
raw_cache = torch.zeros((1, 4, 1, 64), dtype=torch.bfloat16, device=device)
|
||||
compressed_cache = torch.zeros((1, 2, 1, 64), dtype=torch.bfloat16, device=device)
|
||||
norm = SimpleNamespace(
|
||||
weight=torch.zeros(64, dtype=torch.bfloat16, device=device),
|
||||
variance_epsilon=1e-6,
|
||||
)
|
||||
indexer = SimpleNamespace(
|
||||
_metadata=lambda: (raw_metadata, compressed_metadata),
|
||||
skip_topk=True,
|
||||
index_kv_heads=1,
|
||||
use_fused_pre_indexer=False,
|
||||
index_n_heads=1,
|
||||
index_head_dim=64,
|
||||
q_layernorm=norm,
|
||||
k_layernorm=norm,
|
||||
rotary_emb=rope,
|
||||
compress_ratio=4,
|
||||
raw_key_cache=SimpleNamespace(
|
||||
kv_cache=raw_cache, key_cache=raw_cache, rope_position_cache=None
|
||||
),
|
||||
compressed_key_cache=SimpleNamespace(kv_cache=compressed_cache),
|
||||
)
|
||||
keys = torch.arange(1, 6, dtype=torch.bfloat16, device=device)[:, None].expand(
|
||||
5, 64
|
||||
)
|
||||
padded_keys = torch.full((8, 64), torch.nan, dtype=torch.bfloat16, device=device)
|
||||
padded_keys[:5].copy_(keys)
|
||||
indexer_qsa.QSAIndexer.forward(
|
||||
indexer,
|
||||
torch.cat((torch.ones_like(padded_keys), padded_keys), dim=-1),
|
||||
torch.zeros(8, dtype=torch.long, device=device),
|
||||
torch.full((5, 5), -1, dtype=torch.int32, device=device),
|
||||
)
|
||||
torch.testing.assert_close(raw_cache[0, :, 0], keys[[4, 1, 2, 3]])
|
||||
expected_compressed = torch.zeros_like(compressed_cache)
|
||||
expected_compressed[0, 0] = 1
|
||||
torch.testing.assert_close(compressed_cache, expected_compressed)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 4])
|
||||
@pytest.mark.parametrize("num_reqs", [2, 3, 4, 7, 8, 9])
|
||||
|
||||
@@ -338,9 +338,15 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph(
|
||||
("DeepseekV32MTPModel", True, False),
|
||||
("GlmMoeDsaForCausalLM", False, True),
|
||||
("GlmMoeDsaForCausalLM", True, False),
|
||||
("Qwen4ExpForCausalLM", False, True),
|
||||
("Qwen4ExpForCausalLM", True, False),
|
||||
("Qwen4ExpForConditionalGeneration", False, True),
|
||||
("Qwen4ExpForConditionalGeneration", True, False),
|
||||
("Qwen4ExpMTP", False, True),
|
||||
("Qwen4ExpMTP", True, False),
|
||||
],
|
||||
)
|
||||
def test_dsa_breakable_cudagraph_platform_default(
|
||||
def test_breakable_cudagraph_platform_default(
|
||||
monkeypatch, architecture, is_rocm, expected
|
||||
):
|
||||
from vllm.config.vllm import default_breakable_cudagraph_architectures
|
||||
|
||||
Reference in New Issue
Block a user