[Bugfix] Fix PD disagg + MTP correctness for Qwen3.5(GDN) (#47466)
Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -28,6 +28,7 @@ hybrid_ssm_configs=(
|
||||
# GDN (Qwen3.5)
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT=DS GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B"
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT=DS PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B"
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT=DS ENFORCE_EAGER=0 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B VLLM_SERVE_EXTRA_ARGS=--spec-method,mtp,--spec-tokens,1"
|
||||
# Mamba1 (Jamba)
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT=DS GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=ai21labs/AI21-Jamba2-3B VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192"
|
||||
)
|
||||
|
||||
@@ -76,6 +76,7 @@ DECODER_TP_SIZE=${DECODER_TP_SIZE:-1}
|
||||
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.2}
|
||||
PREFILL_BLOCK_SIZE=${PREFILL_BLOCK_SIZE:-128}
|
||||
DECODE_BLOCK_SIZE=${DECODE_BLOCK_SIZE:-128}
|
||||
ENFORCE_EAGER=${ENFORCE_EAGER:-1}
|
||||
# Comma-separated extra args for vllm serve (e.g. --max-model-len,2048)
|
||||
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
|
||||
|
||||
@@ -157,11 +158,13 @@ run_tests_for_model() {
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=$SIDE_CHANNEL_PORT \
|
||||
vllm serve $model_name \
|
||||
--port $PORT \
|
||||
--enforce-eager \
|
||||
--block-size ${PREFILL_BLOCK_SIZE} \
|
||||
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
|
||||
--tensor-parallel-size $PREFILLER_TP_SIZE \
|
||||
--kv-transfer-config '$KV_CONFIG_P'"
|
||||
if [[ "$ENFORCE_EAGER" == "1" ]]; then
|
||||
BASE_CMD="${BASE_CMD} --enforce-eager"
|
||||
fi
|
||||
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
|
||||
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
|
||||
for arg in "${extra_args[@]}"; do
|
||||
@@ -206,10 +209,12 @@ run_tests_for_model() {
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=$SIDE_CHANNEL_PORT \
|
||||
vllm serve $model_name \
|
||||
--port $PORT \
|
||||
--enforce-eager \
|
||||
--block-size ${DECODE_BLOCK_SIZE} \
|
||||
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
|
||||
--kv-transfer-config '$KV_CONFIG_D'"
|
||||
if [[ "$ENFORCE_EAGER" == "1" ]]; then
|
||||
BASE_CMD="${BASE_CMD} --enforce-eager"
|
||||
fi
|
||||
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
|
||||
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
|
||||
for arg in "${extra_args[@]}"; do
|
||||
|
||||
@@ -340,10 +340,12 @@ def _combine_sampled_and_draft_tokens_kernel(
|
||||
# Handling prefill tokens. No sampled or draft tokens.
|
||||
return
|
||||
|
||||
if NUM_NEW_SAMPLED_TOKENS > 0:
|
||||
# Keep prompt-tail slots intact; only rewrite generated-token slots.
|
||||
first_logit_seq_pos = seq_len - num_logits
|
||||
if NUM_NEW_SAMPLED_TOKENS > 0 and first_logit_seq_pos >= prefill_len:
|
||||
# Write the last sampled token ID to input_ids.
|
||||
last_token_id = tl.load(last_sampled_tokens_ptr + req_state_idx)
|
||||
tl.store(input_ids_ptr + query_end - num_logits, last_token_id)
|
||||
tl.store(input_ids_ptr + logits_start, last_token_id)
|
||||
|
||||
# Write the draft tokens (if any) to input_ids.
|
||||
if num_draft_tokens > 0:
|
||||
|
||||
@@ -255,11 +255,16 @@ class MambaHybridModelState(DefaultModelState):
|
||||
# GDN uses >= 0 to select spec-decode rows, so non-decode rows
|
||||
# need the -1 sentinel rather than a raw zero draft count.
|
||||
num_decode_draft_tokens_np = np.full(num_reqs, -1, dtype=np.int32)
|
||||
if input_batch.num_draft_tokens_per_req is not None:
|
||||
has_draft_tokens = input_batch.num_draft_tokens_per_req > 0
|
||||
spec_decode_mask = has_draft_tokens & ~input_batch.is_prefilling_np
|
||||
num_draft_tokens_per_req = input_batch.num_draft_tokens_per_req
|
||||
if num_draft_tokens_per_req is not None:
|
||||
# A row is a spec-decode row only when its whole prompt is already
|
||||
# computed, i.e. exactly one non-draft (decode) token is scheduled.
|
||||
is_decode = (
|
||||
input_batch.num_scheduled_tokens == num_draft_tokens_per_req + 1
|
||||
)
|
||||
spec_decode_mask = (num_draft_tokens_per_req > 0) & is_decode
|
||||
num_decode_draft_tokens_np[: input_batch.num_reqs] = np.where(
|
||||
spec_decode_mask, input_batch.num_draft_tokens_per_req, -1
|
||||
spec_decode_mask, num_draft_tokens_per_req, -1
|
||||
)
|
||||
num_decode_draft_tokens_cpu = torch.from_numpy(num_decode_draft_tokens_np)
|
||||
|
||||
|
||||
@@ -110,16 +110,6 @@ class RequestState:
|
||||
self.num_computed_tokens_np[req_idx] = num_computed_tokens
|
||||
self.num_computed_tokens.stage_write_elem(req_idx, num_computed_tokens)
|
||||
|
||||
if 0 < num_computed_tokens <= prefill_len:
|
||||
# For PD disagg or resumed requests: set last_sampled to the last
|
||||
# computed token so the first decode step gets the right input_id.
|
||||
# For fresh prefill requests (num_computed_tokens == 0) the tensor
|
||||
# is not read by combine_sampled_and_draft_tokens so we skip the
|
||||
# write. Use a slice assignment rather than scalar indexing so the
|
||||
# write is dispatched through fill_ without a host/device sync.
|
||||
self.last_sampled_tokens[req_idx : req_idx + 1] = all_token_ids[
|
||||
num_computed_tokens - 1
|
||||
]
|
||||
self.draft_tokens[req_idx].zero_()
|
||||
|
||||
def apply_staged_writes(self) -> None:
|
||||
|
||||
@@ -2208,10 +2208,7 @@ class GPUModelRunner(
|
||||
req_idx = self.input_batch.req_id_to_index[req_id]
|
||||
draft_len = len(draft_token_ids)
|
||||
num_draft_tokens[req_idx] = draft_len
|
||||
if (
|
||||
self.input_batch.num_computed_tokens_cpu[req_idx]
|
||||
>= self.input_batch.num_prompt_tokens[req_idx]
|
||||
):
|
||||
if num_scheduled_tokens[req_idx] == draft_len + 1:
|
||||
num_decode_draft_tokens[req_idx] = draft_len
|
||||
spec_decode_metadata = self._calc_spec_decode_metadata(
|
||||
num_draft_tokens, cu_num_tokens
|
||||
|
||||
Reference in New Issue
Block a user