From aa15ca17194261c0e6406b7331cb88ba9b4fcd1e Mon Sep 17 00:00:00 2001
From: hiyouga <467089858@qq.com>
Date: Tue, 9 Jul 2024 23:24:24 +0800
Subject: [PATCH] fix #4742

Former-commit-id: 2f09520c0d5039a5a8be310ab668272cb4dc1bd3
---
 src/llamafactory/chat/vllm_engine.py  | 2 +-
 src/llamafactory/train/dpo/trainer.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/llamafactory/chat/vllm_engine.py b/src/llamafactory/chat/vllm_engine.py
index ac7ac769..8dc7214a 100644
--- a/src/llamafactory/chat/vllm_engine.py
+++ b/src/llamafactory/chat/vllm_engine.py
@@ -135,7 +135,7 @@ class VllmEngine(BaseEngine):
             if is_vllm_version_greater_than_0_5_1():
                 multi_modal_data = {"image": pixel_values}
             elif is_vllm_version_greater_than_0_5():
-                multi_modal_data = ImagePixelData(image=pixel_values)   
+                multi_modal_data = ImagePixelData(image=pixel_values)
             else:  # TODO: remove vllm 0.4.3 support
                 multi_modal_data = MultiModalData(type=MultiModalData.Type.IMAGE, data=pixel_values)
         else:
diff --git a/src/llamafactory/train/dpo/trainer.py b/src/llamafactory/train/dpo/trainer.py
index e45467d6..9c07df66 100644
--- a/src/llamafactory/train/dpo/trainer.py
+++ b/src/llamafactory/train/dpo/trainer.py
@@ -54,6 +54,7 @@ class CustomDPOTrainer(DPOTrainer):
                 disable_dropout_in_model(ref_model)
 
         self.finetuning_args = finetuning_args
+        self.f_divergence_type = "reverse_kl"
         self.reference_free = False
         self.use_dpo_data_collator = True  # hack to avoid warning
         self.generate_during_eval = False  # disable at evaluation