From b37bb592eca7bd07eea374e92c6a9e0979623355 Mon Sep 17 00:00:00 2001
From: BUAADreamer <1428195643@qq.com>
Date: Sun, 29 Sep 2024 22:40:43 +0800
Subject: [PATCH] fix constants

Former-commit-id: bec1cb8d55d01ac8b70b9bacd92a227b48cf8336
---
 src/llamafactory/data/template.py    | 27 +++++++++++++++-------
 src/llamafactory/extras/constants.py | 34 ++++++++++++++++++++--------
 2 files changed, 44 insertions(+), 17 deletions(-)

diff --git a/src/llamafactory/data/template.py b/src/llamafactory/data/template.py
index a23e0e44..99cbf86b 100644
--- a/src/llamafactory/data/template.py
+++ b/src/llamafactory/data/template.py
@@ -782,14 +782,6 @@ _register_template(
         ]
     ),
     format_system=StringFormatter(slots=["<|start_header_id|>system<|end_header_id|>\n\n{{content}}<|eot_id|>"]),
-    format_observation=StringFormatter(
-        slots=[
-            (
-                "<|start_header_id|>tool<|end_header_id|>\n\n{{content}}<|eot_id|>"
-                "<|start_header_id|>assistant<|end_header_id|>\n\n"
-            )
-        ]
-    ),
     format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
     stop_words=["<|eot_id|>"],
     replace_eos=True,
@@ -806,6 +798,17 @@ _register_template(
 )
 
 
+_register_template(
+    name="llava_next_qwen",
+    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
+    format_separator=EmptyFormatter(slots=["\n"]),
+    stop_words=["<|im_end|>"],
+    replace_eos=True,
+    replace_jinja_template=False,
+    mm_plugin=get_mm_plugin(name="llava_next", image_token="<image>"),
+)
+
+
 _register_template(
     name="llava_next_yi",
     format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
@@ -828,6 +831,14 @@ _register_template(
 )
 
 
+_register_template(
+    name="llava_next_video_mistral",
+    format_user=StringFormatter(slots=["[INST] {{content}} [/INST]"]),
+    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
+    mm_plugin=get_mm_plugin(name="llava_next_video", image_token="<image>", video_token="<video>"),
+)
+
+
 _register_template(
     name="llava_next_video_yi",
     format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
diff --git a/src/llamafactory/extras/constants.py b/src/llamafactory/extras/constants.py
index 1be3940e..02bb9066 100644
--- a/src/llamafactory/extras/constants.py
+++ b/src/llamafactory/extras/constants.py
@@ -854,12 +854,6 @@ register_model_group(
         "LLaVA-NeXT-LLaMA3-8B-Chat": {
             DownloadSource.DEFAULT: "llava-hf/llama3-llava-next-8b-hf",
         },
-        "LLaVA-NeXT-LLaMA3-72B-Chat": {
-            DownloadSource.DEFAULT: "llava-hf/llava-next-72b-hf",
-        },
-        "LLaVA-NeXT-LLaMA3-110B-Chat": {
-            DownloadSource.DEFAULT: "llava-hf/llava-next-110b-hf",
-        },
     },
     template="llava_next_llama3",
     vision=True,
@@ -877,6 +871,20 @@ register_model_group(
 )
 
 
+register_model_group(
+    models={
+        "LLaVA-NeXT-Qwen-1.5-72B-Chat": {
+            DownloadSource.DEFAULT: "llava-hf/llava-next-72b-hf",
+        },
+        "LLaVA-NeXT-Qwen-1.5-110B-Chat": {
+            DownloadSource.DEFAULT: "llava-hf/llava-next-110b-hf",
+        },
+    },
+    template="llava_next_qwen",
+    vision=True,
+)
+
+
 register_model_group(
     models={
         "LLaVA-NeXT-Yi-34B-Chat": {
@@ -893,9 +901,6 @@ register_model_group(
         "LLaVA-NeXT-Video-7B-Chat": {
             DownloadSource.DEFAULT: "llava-hf/LLaVA-NeXT-Video-7B-hf",
         },
-        "LLaVA-NeXT-Video-7B-32k-Chat": {
-            DownloadSource.DEFAULT: "llava-hf/LLaVA-NeXT-Video-7B-32K-hf",
-        },
         "LLaVA-NeXT-Video-7B-DPO-Chat": {
             DownloadSource.DEFAULT: "llava-hf/LLaVA-NeXT-Video-7B-DPO-hf",
         },
@@ -905,6 +910,17 @@ register_model_group(
 )
 
 
+register_model_group(
+    models={
+        "LLaVA-NeXT-Video-Mistral-7B-32k-Chat": {
+            DownloadSource.DEFAULT: "llava-hf/LLaVA-NeXT-Video-7B-32K-hf",
+        },
+    },
+    template="llava_next_video_mistral",
+    vision=True,
+)
+
+
 register_model_group(
     models={
         "LLaVA-NeXT-Video-Yi-34B-Chat": {