zijie-tian/nano-vllm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
GeeeekExplorer
2025-06-11 21:17:23 +08:00
parent 08c84ec08d
commit fee58d44e4
5 changed files with 4 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
nanovllm/engine/llm_engine.py
View File

@@ -52,7 +52,7 @@ class LLMEngine:
desc="Generating",
dynamic_ncols=True,
)
if not isinstance(SamplingParams, list):
if not isinstance(sampling_params, list):
sampling_params = [sampling_params] * len(prompts)
for prompt, sp in zip(prompts, sampling_params):
self.add_request(prompt, sp)

2
nanovllm/engine/model_runner.py
View File

@@ -170,7 +170,7 @@ class ModelRunner:
context_lens = torch.zeros(max_bs, dtype=torch.int32)
block_tables = torch.zeros(max_bs, max_num_blocks, dtype=torch.int32)
outputs = torch.zeros(max_bs, hf_config.hidden_size)
self.graph_bs = [1, 2, 4, 8, 16] + list(range(16, max_bs + 1, 16))
self.graph_bs = [1, 2, 4, 8] + list(range(16, max_bs + 1, 16))
self.graphs = {}
self.graph_pool = None