[Bugfix] Fix PD disagg + MTP correctness for Qwen3.5(GDN) (#47466)

Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Dakai An
2026-07-07 13:51:22 +00:00
committed by GitHub
co-authored by Nick Hill
parent 65a7b46284
commit 65dcde1695
6 changed files with 22 additions and 22 deletions
@@ -28,6 +28,7 @@ hybrid_ssm_configs=(
# GDN (Qwen3.5)
"VLLM_SSM_CONV_STATE_LAYOUT=DS GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B"
"VLLM_SSM_CONV_STATE_LAYOUT=DS PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B"
"VLLM_SSM_CONV_STATE_LAYOUT=DS ENFORCE_EAGER=0 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B VLLM_SERVE_EXTRA_ARGS=--spec-method,mtp,--spec-tokens,1"
# Mamba1 (Jamba)
"VLLM_SSM_CONV_STATE_LAYOUT=DS GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=ai21labs/AI21-Jamba2-3B VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192"
)
@@ -76,6 +76,7 @@ DECODER_TP_SIZE=${DECODER_TP_SIZE:-1}
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.2}
PREFILL_BLOCK_SIZE=${PREFILL_BLOCK_SIZE:-128}
DECODE_BLOCK_SIZE=${DECODE_BLOCK_SIZE:-128}
ENFORCE_EAGER=${ENFORCE_EAGER:-1}
# Comma-separated extra args for vllm serve (e.g. --max-model-len,2048)
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
@@ -157,11 +158,13 @@ run_tests_for_model() {
VLLM_NIXL_SIDE_CHANNEL_PORT=$SIDE_CHANNEL_PORT \
vllm serve $model_name \
--port $PORT \
--enforce-eager \
--block-size ${PREFILL_BLOCK_SIZE} \
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
--tensor-parallel-size $PREFILLER_TP_SIZE \
--kv-transfer-config '$KV_CONFIG_P'"
if [[ "$ENFORCE_EAGER" == "1" ]]; then
BASE_CMD="${BASE_CMD} --enforce-eager"
fi
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
for arg in "${extra_args[@]}"; do
@@ -206,10 +209,12 @@ run_tests_for_model() {
VLLM_NIXL_SIDE_CHANNEL_PORT=$SIDE_CHANNEL_PORT \
vllm serve $model_name \
--port $PORT \
--enforce-eager \
--block-size ${DECODE_BLOCK_SIZE} \
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
--kv-transfer-config '$KV_CONFIG_D'"
if [[ "$ENFORCE_EAGER" == "1" ]]; then
BASE_CMD="${BASE_CMD} --enforce-eager"
fi
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
for arg in "${extra_args[@]}"; do
+4 -2
View File
@@ -340,10 +340,12 @@ def _combine_sampled_and_draft_tokens_kernel(
# Handling prefill tokens. No sampled or draft tokens.
return
if NUM_NEW_SAMPLED_TOKENS > 0:
# Keep prompt-tail slots intact; only rewrite generated-token slots.
first_logit_seq_pos = seq_len - num_logits
if NUM_NEW_SAMPLED_TOKENS > 0 and first_logit_seq_pos >= prefill_len:
# Write the last sampled token ID to input_ids.
last_token_id = tl.load(last_sampled_tokens_ptr + req_state_idx)
tl.store(input_ids_ptr + query_end - num_logits, last_token_id)
tl.store(input_ids_ptr + logits_start, last_token_id)
# Write the draft tokens (if any) to input_ids.
if num_draft_tokens > 0:
@@ -255,11 +255,16 @@ class MambaHybridModelState(DefaultModelState):
# GDN uses >= 0 to select spec-decode rows, so non-decode rows
# need the -1 sentinel rather than a raw zero draft count.
num_decode_draft_tokens_np = np.full(num_reqs, -1, dtype=np.int32)
if input_batch.num_draft_tokens_per_req is not None:
has_draft_tokens = input_batch.num_draft_tokens_per_req > 0
spec_decode_mask = has_draft_tokens & ~input_batch.is_prefilling_np
num_draft_tokens_per_req = input_batch.num_draft_tokens_per_req
if num_draft_tokens_per_req is not None:
# A row is a spec-decode row only when its whole prompt is already
# computed, i.e. exactly one non-draft (decode) token is scheduled.
is_decode = (
input_batch.num_scheduled_tokens == num_draft_tokens_per_req + 1
)
spec_decode_mask = (num_draft_tokens_per_req > 0) & is_decode
num_decode_draft_tokens_np[: input_batch.num_reqs] = np.where(
spec_decode_mask, input_batch.num_draft_tokens_per_req, -1
spec_decode_mask, num_draft_tokens_per_req, -1
)
num_decode_draft_tokens_cpu = torch.from_numpy(num_decode_draft_tokens_np)
-10
View File
@@ -110,16 +110,6 @@ class RequestState:
self.num_computed_tokens_np[req_idx] = num_computed_tokens
self.num_computed_tokens.stage_write_elem(req_idx, num_computed_tokens)
if 0 < num_computed_tokens <= prefill_len:
# For PD disagg or resumed requests: set last_sampled to the last
# computed token so the first decode step gets the right input_id.
# For fresh prefill requests (num_computed_tokens == 0) the tensor
# is not read by combine_sampled_and_draft_tokens so we skip the
# write. Use a slice assignment rather than scalar indexing so the
# write is dispatched through fill_ without a host/device sync.
self.last_sampled_tokens[req_idx : req_idx + 1] = all_token_ids[
num_computed_tokens - 1
]
self.draft_tokens[req_idx].zero_()
def apply_staged_writes(self) -> None:
+1 -4
View File
@@ -2208,10 +2208,7 @@ class GPUModelRunner(
req_idx = self.input_batch.req_id_to_index[req_id]
draft_len = len(draft_token_ids)
num_draft_tokens[req_idx] = draft_len
if (
self.input_batch.num_computed_tokens_cpu[req_idx]
>= self.input_batch.num_prompt_tokens[req_idx]
):
if num_scheduled_tokens[req_idx] == draft_len + 1:
num_decode_draft_tokens[req_idx] = draft_len
spec_decode_metadata = self._calc_spec_decode_metadata(
num_draft_tokens, cu_num_tokens