support vllm 0.6.0

This commit is contained in:
hiyouga
2024-09-08 02:26:20 +08:00
parent b332908ab4
commit b6681d7198
5 changed files with 12 additions and 51 deletions

View File

@@ -103,7 +103,6 @@ class HuggingfaceEngine(BaseEngine):
prompt_ids, _ = template.mm_plugin.process_token_ids(
prompt_ids, None, mm_input_dict["images"], mm_input_dict["videos"], tokenizer, processor
)
prompt_length = len(prompt_ids)
inputs = torch.tensor([prompt_ids], device=model.device)
attention_mask = torch.ones_like(inputs, dtype=torch.bool)