zijie-tian/nano-vllm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[feat] Added chunked prefill and kvcache offload mechenism.

Browse Source
This commit is contained in:
Zijie Tian
2025-12-10 03:47:37 +08:00
parent 204fe2b38f
commit 0b6f19242d
25 changed files with 4414 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
bench_vllm.py
View File

@@ -47,16 +47,16 @@ def main():
print("Prefill Benchmark")
print("=" * 60)
bench_prefill(llm, num_seqs=1, input_len=1024)
bench_prefill(llm, num_seqs=1, input_len=2048)
bench_prefill(llm, num_seqs=1, input_len=4095)
bench_prefill(llm, num_seqs=16, input_len=1024)
bench_prefill(llm, num_seqs=64, input_len=1024)
# bench_prefill(llm, num_seqs=1, input_len=2048)
# bench_prefill(llm, num_seqs=1, input_len=4095)
# bench_prefill(llm, num_seqs=16, input_len=1024)
# bench_prefill(llm, num_seqs=64, input_len=1024)
print("=" * 60)
print("Decode Benchmark")
print("=" * 60)
bench_decode(llm, num_seqs=1, max_input_len=1024, max_output_len=1024)
bench_decode(llm, num_seqs=256, max_input_len=1024, max_output_len=1024)
# bench_decode(llm, num_seqs=256, max_input_len=1024, max_output_len=1024)
if __name__ == "__main__":