refactor data preprocessing, fix mllm rlhf

This commit is contained in:
hiyouga
2024-05-24 04:08:25 +08:00
parent a506f3628b
commit 3a023bca2a
15 changed files with 572 additions and 464 deletions

View File

@@ -98,7 +98,7 @@ class VllmEngine(BaseEngine):
and image is not None
and not hasattr(self.processor, "image_seq_length")
and IMAGE_TOKEN not in messages[0]["content"]
): # llava case
): # llava-like models
messages[0]["content"] = IMAGE_TOKEN * self.image_feature_size + messages[0]["content"]
paired_messages = messages + [{"role": "assistant", "content": ""}]