zijie-tian/nano-vllm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[feat] Finished offload. Still need optimize performance.

Browse Source
This commit is contained in:
Zijie Tian
2025-12-12 02:27:40 +08:00
parent 9b8165af5a
commit 61edb8a344
3 changed files with 72 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
bench_offload.py
View File

@@ -4,10 +4,10 @@ from random import randint, seed
from nanovllm import LLM, SamplingParams
def bench_decode(llm, num_seqs, max_input_len, max_output_len):
def bench_decode(llm, num_seqs, input_len, max_output_len):
"""Benchmark decode performance (original test)"""
seed(0)
prompt_token_ids = [[randint(0, 10000) for _ in range(randint(100, max_input_len))] for _ in range(num_seqs)]
prompt_token_ids = [[randint(0, 10000) for _ in range(randint(100, input_len))] for _ in range(num_seqs)]
sampling_params = [SamplingParams(temperature=0.6, ignore_eos=True, max_tokens=randint(100, max_output_len)) for _ in range(num_seqs)]
t = time.time()
@@ -54,13 +54,13 @@ def main():
# bench_prefill(llm, num_seqs=1, input_len=1024)
# bench_prefill(llm, num_seqs=1, input_len=2048)
# bench_prefill(llm, num_seqs=1, input_len=4096)
bench_prefill(llm, num_seqs=1, input_len=8192)
bench_prefill(llm, num_seqs=1, input_len=64 * 1024)
print("=" * 60)
print("Decode Benchmark (CPU Offload)")
print("=" * 60)
bench_decode(llm, num_seqs=1, max_input_len=1024, max_output_len=128)
# bench_decode(llm, num_seqs=1, max_input_len=2048, max_output_len=128)
bench_decode(llm, num_seqs=1, input_len=64 * 1024, max_output_len=128)
# bench_decode(llm, num_seqs=1, input_len=2048, max_output_len=128)
if __name__ == "__main__":