[Qwen3.8-Flash-Next] Remove torch.compile for NVIDIA implementation (#55272)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Thien Tran
2026-09-06 21:57:56 -07:00
committed by GitHub
parent 9cc7793e32
commit d9105ea800
13 changed files with 165 additions and 185 deletions
+37 -14
View File
@@ -147,9 +147,9 @@ def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
model_state.uses_ngram_embedding = True
model_state.ngram_context_len = 3
model_state.ngram_eos_token_id = 99
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
model_state.ngram_context = torch.empty((8, 3), dtype=torch.int32)
model_state.ngram_context_offsets = torch.arange(-3, 0, dtype=torch.int64)
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
model_state.ple_query_start_loc = torch.empty(9, dtype=torch.int32)
input_batch = SimpleNamespace(
num_reqs=2,
@@ -167,34 +167,57 @@ def test_qwen4_exp_model_state_prepares_ngram_context() -> None:
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
model_inputs = model_state.prepare_inputs(input_batch, req_states)
expected_query_start_loc = torch.full((9,), 3, dtype=torch.int32)
expected_query_start_loc[0] = 0
expected_query_start_loc[1] = 2
torch.testing.assert_close(
model_inputs["query_start_loc"],
torch.tensor([0, 2, 3, 3], dtype=torch.int32),
model_inputs["query_start_loc"], expected_query_start_loc
)
expected_context = torch.full((8, 3), 99, dtype=torch.int32)
expected_context[:2] = torch.tensor([[99, 99, 20], [1, 2, 3]])
torch.testing.assert_close(model_inputs["ngram_context"], expected_context)
# Retain the views to detect reallocations as the request layout changes.
query_start_loc = model_inputs["query_start_loc"]
ngram_context = model_inputs["ngram_context"]
input_batch.num_reqs = 1
input_batch.num_reqs_after_padding = 1
input_batch.idx_mapping = torch.tensor([0])
input_batch.query_start_loc = torch.tensor([0, 3], dtype=torch.int32)
with patch.object(MambaHybridModelState, "prepare_inputs", return_value={}):
model_inputs = model_state.prepare_inputs(input_batch, req_states)
expected_query_start_loc.fill_(3)
expected_query_start_loc[0] = 0
torch.testing.assert_close(
model_inputs["ngram_context"],
torch.tensor([[99, 99, 20], [1, 2, 3], [99, 99, 99]], dtype=torch.int32),
model_inputs["query_start_loc"], expected_query_start_loc
)
expected_context.fill_(99)
expected_context[0] = torch.tensor([1, 2, 3])
torch.testing.assert_close(model_inputs["ngram_context"], expected_context)
assert model_inputs["query_start_loc"].data_ptr() == query_start_loc.data_ptr()
assert model_inputs["ngram_context"].data_ptr() == ngram_context.data_ptr()
def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
model_state = object.__new__(Qwen4ExpModelState)
model_state.uses_ngram_embedding = True
model_state.ngram_eos_token_id = 99
model_state.ngram_context = torch.empty((4, 3), dtype=torch.int32)
model_state.ple_query_start_loc = torch.empty(5, dtype=torch.int32)
model_state.ngram_context = torch.empty((8, 3), dtype=torch.int32)
model_state.ple_query_start_loc = torch.empty(9, dtype=torch.int32)
with patch.object(MambaHybridModelState, "prepare_dummy_inputs", return_value={}):
first = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
first = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=4)
# Dummy runs establish the addresses used during CUDA graph capture.
query_start_loc_ptr = first["query_start_loc"].data_ptr()
ngram_context_ptr = first["ngram_context"].data_ptr()
second = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=8)
second = model_state.prepare_dummy_inputs(num_reqs=3, num_tokens=4)
expected_query_start_loc = torch.full((9,), 4, dtype=torch.int32)
expected_query_start_loc[:4] = torch.tensor([0, 1, 2, 4], dtype=torch.int32)
torch.testing.assert_close(second["query_start_loc"], expected_query_start_loc)
torch.testing.assert_close(
second["query_start_loc"], torch.tensor([0, 2, 5, 8], dtype=torch.int32)
)
torch.testing.assert_close(
second["ngram_context"], torch.full((3, 3), 99, dtype=torch.int32)
second["ngram_context"], torch.full((8, 3), 99, dtype=torch.int32)
)
assert second["query_start_loc"].data_ptr() == query_start_loc_ptr
assert second["ngram_context"].data_ptr() == ngram_context_ptr
+12
View File
@@ -498,6 +498,17 @@ def _ngram_hash_params(device: torch.device, context_len: int) -> dict:
([3], [1], [[11]], 20),
([4, 4], [0, 7], [[11, 12], [13, 14]], 20),
([4, 0, 3], [], [[11, 12], [13, 14], [15, 16]], 20),
(
[3, 2, 0, 0],
[],
[
[11, 12],
[13, 14],
[_NGRAM_EOS_TOKEN_ID, _NGRAM_EOS_TOKEN_ID],
[_NGRAM_EOS_TOKEN_ID, _NGRAM_EOS_TOKEN_ID],
],
20,
),
(
[1, 33, 2],
[5, 32],
@@ -528,6 +539,7 @@ def _ngram_hash_params(device: torch.device, context_len: int) -> dict:
"bigram-only",
"power-of-two",
"empty-request",
"trailing-padded-requests",
"three-requests",
"six-requests",
"four-gram",
+66 -2
View File
@@ -45,7 +45,6 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
indexer = SimpleNamespace(
skip_topk=True,
_metadata=lambda: (raw_metadata, compressed_metadata),
index_qk_proj=lambda hidden: (torch.zeros(2, 2), None),
index_n_heads=1,
index_kv_heads=1,
index_head_dim=1,
@@ -80,7 +79,7 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
actual = indexer_qsa.QSAIndexer.forward(
indexer,
torch.zeros(2, 4),
torch.zeros(2, 2),
torch.tensor([7, 8]),
rows,
)
@@ -446,6 +445,71 @@ def test_qsa_compressed_metadata_keeps_dummy_slots_inert() -> None:
assert metadata.k_work_metadata.tolist() == [[0, 0], [2, 0], [2, 1], [-1, -1]]
@requires_qsa_kernels
@pytest.mark.usefixtures("default_vllm_config")
def test_qsa_unfused_cache_update_ignores_padded_qk() -> None:
"""Padded projected Q/K rows must not affect either side cache."""
from vllm.model_executor.layers.rotary_embedding import get_rope
device = torch.device("cuda")
# Five tokens complete one compressed group and retain four keys in the ring.
raw_metadata = SimpleNamespace(
num_actual_tokens=5,
slot_mapping=torch.tensor([-1, 1, 2, 3, 0], device=device),
block_table=torch.zeros((1, 1), dtype=torch.int32, device=device),
token_to_req=torch.zeros(5, dtype=torch.int32, device=device),
query_start_loc=torch.tensor([0, 5], dtype=torch.int32, device=device),
logical_positions=torch.arange(5, device=device),
)
compressed_metadata = SimpleNamespace(
slot_mapping=torch.tensor([-1, -1, -1, 0, -1], device=device),
)
with torch.device(device):
rope = get_rope(
head_size=128,
max_position=32,
rope_parameters={"rope_type": "default", "partial_rotary_factor": 0.5},
dtype=torch.bfloat16,
)
raw_cache = torch.zeros((1, 4, 1, 64), dtype=torch.bfloat16, device=device)
compressed_cache = torch.zeros((1, 2, 1, 64), dtype=torch.bfloat16, device=device)
norm = SimpleNamespace(
weight=torch.zeros(64, dtype=torch.bfloat16, device=device),
variance_epsilon=1e-6,
)
indexer = SimpleNamespace(
_metadata=lambda: (raw_metadata, compressed_metadata),
skip_topk=True,
index_kv_heads=1,
use_fused_pre_indexer=False,
index_n_heads=1,
index_head_dim=64,
q_layernorm=norm,
k_layernorm=norm,
rotary_emb=rope,
compress_ratio=4,
raw_key_cache=SimpleNamespace(
kv_cache=raw_cache, key_cache=raw_cache, rope_position_cache=None
),
compressed_key_cache=SimpleNamespace(kv_cache=compressed_cache),
)
keys = torch.arange(1, 6, dtype=torch.bfloat16, device=device)[:, None].expand(
5, 64
)
padded_keys = torch.full((8, 64), torch.nan, dtype=torch.bfloat16, device=device)
padded_keys[:5].copy_(keys)
indexer_qsa.QSAIndexer.forward(
indexer,
torch.cat((torch.ones_like(padded_keys), padded_keys), dim=-1),
torch.zeros(8, dtype=torch.long, device=device),
torch.full((5, 5), -1, dtype=torch.int32, device=device),
)
torch.testing.assert_close(raw_cache[0, :, 0], keys[[4, 1, 2, 3]])
expected_compressed = torch.zeros_like(compressed_cache)
expected_compressed[0, 0] = 1
torch.testing.assert_close(compressed_cache, expected_compressed)
@requires_qsa_kernels
@pytest.mark.parametrize("compress_ratio", [1, 4])
@pytest.mark.parametrize("num_reqs", [2, 3, 4, 7, 8, 9])