[GLM 5.3 Perf] Enable fused multi-step decode, 13.3% E2E throughput improvement for concurrency 1 (#57443)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -39,6 +39,7 @@ def _make_fake_speculator(
|
||||
and a draft model.
|
||||
"""
|
||||
fake_input_buffers = SimpleNamespace(
|
||||
positions=torch.arange(max_num_tokens, dtype=torch.int64),
|
||||
query_start_loc=torch.zeros(max_num_reqs + 1, dtype=torch.int32),
|
||||
seq_lens=torch.zeros(max_num_reqs, dtype=torch.int32),
|
||||
dcp_local_seq_lens=torch.zeros(max_num_reqs, dtype=torch.int32),
|
||||
@@ -113,6 +114,7 @@ def test_build_draft_attn_metadata_sets_seq_lens_cpu_upper_bound():
|
||||
|
||||
captured = _run_build(fake, num_reqs=3, num_reqs_padded=4, base=base, step=2)
|
||||
|
||||
assert torch.equal(captured["positions"], fake.input_buffers.positions[:4])
|
||||
bound = captured["seq_lens_cpu_upper_bound"]
|
||||
assert isinstance(bound, torch.Tensor), (
|
||||
"seq_lens_cpu_upper_bound must be a tensor, not None"
|
||||
|
||||
Reference in New Issue
Block a user